diff --git a/kafka/docker-compose.yml b/kafka/docker-compose.yml index f36dbc6..365b0f0 100644 --- a/kafka/docker-compose.yml +++ b/kafka/docker-compose.yml @@ -1,82 +1,93 @@ -version: '3' services: - zookeeper-1: - image: confluentinc/cp-zookeeper:latest - container_name: zookeeper-1 + broker1: + image: apache/kafka:latest + container_name: broker1 environment: - ZOOKEEPER_SERVER_ID: 1 - ZOOKEEPER_CLIENT_PORT: 22181 - ZOOKEEPER_TICK_TIME: 2000 - ports: - - "22181:22181" - volumes: - - /data/kafka/zookeeper/data:/var/lib/zookeeper/data - - /data/kafka/zookeeper/log:/var/lib/zookeeper/log - - broker-1: - image: confluentinc/cp-kafka:latest - container_name: broker-1 - ports: - - "19092:19092" - depends_on: - - zookeeper-1 - environment: - KAFKA_BROKER_ID: 1 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:22181' - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-1:19092,PLAINTEXT_HOST://localhost:29092 - KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_KRAFT_CLUSTER_ID: kafka-cluster-3-nodes # Define your cluster ID at .env file + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: "broker,controller" + KAFKA_LISTENERS: INTERNAL://:9092,CONTROLLER://:9093,EXTERNAL://:19092 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker1:9092,EXTERNAL://localhost:19092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LOG_DIRS: /var/lib/kafka/data + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2 + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@broker1:9093,2@broker2:9093,3@broker3:9093 volumes: - /data/kafka/broker/01:/var/lib/kafka/data - - broker-2: - image: confluentinc/cp-kafka:latest - container_name: broker-2 + networks: + - kafka-net ports: - - "19093:19093" - depends_on: - - zookeeper-1 + - "19092:19092" # Expose external port for broker1 + restart: unless-stopped + + broker2: + image: apache/kafka:latest + container_name: broker2 environment: - KAFKA_BROKER_ID: 2 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:22181' - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-2:19093,PLAINTEXT_HOST://localhost:29093 - KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_KRAFT_CLUSTER_ID: kafka-cluster-3-nodes # Define your cluster ID at .env file + KAFKA_NODE_ID: 2 + KAFKA_PROCESS_ROLES: "broker,controller" + KAFKA_LISTENERS: INTERNAL://:9092,CONTROLLER://:9093,EXTERNAL://:19092 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker2:9092,EXTERNAL://localhost:19092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LOG_DIRS: /var/lib/kafka/data + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2 + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@broker1:9093,2@broker2:9093,3@broker3:9093 volumes: - /data/kafka/broker/02:/var/lib/kafka/data - - broker-3: - image: confluentinc/cp-kafka:latest - container_name: broker-3 + networks: + - kafka-net ports: - - "19094:19094" - depends_on: - - zookeeper-1 + - "19093:19092" # Expose external port for broker2 + restart: unless-stopped + + broker3: + image: apache/kafka:latest + container_name: broker3 environment: - KAFKA_BROKER_ID: 3 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:22181' - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-3:19094,PLAINTEXT_HOST://localhost:29094 - KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_KRAFT_CLUSTER_ID: kafka-cluster-3-nodes # Define your cluster ID at .env file + KAFKA_NODE_ID: 3 + KAFKA_PROCESS_ROLES: "broker,controller" + KAFKA_LISTENERS: INTERNAL://:9092,CONTROLLER://:9093,EXTERNAL://:19092 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker3:9092,EXTERNAL://localhost:19092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LOG_DIRS: /var/lib/kafka/data + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2 + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@broker1:9093,2@broker2:9093,3@broker3:9093 volumes: - /data/kafka/broker/03:/var/lib/kafka/data + networks: + - kafka-net + ports: + - "19094:19092" # Expose external port for broker3 + restart: unless-stopped - kafka-ui: - container_name: kafka-ui - image: provectuslabs/kafka-ui:latest - ports: - - 18080:8080 # Changed to avoid port clash with akhq - depends_on: - - broker-1 - - broker-2 - - broker-3 - environment: - KAFKA_CLUSTERS_0_NAME: local - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: broker-1:19092,broker-2:19093,broker-3:19094 - # KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schema-registry:8081 - # KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: connect - # KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: http://kafka-connect:8083 - DYNAMIC_CONFIG_ENABLED: 'true' \ No newline at end of file + kafdrop: + image: obsidiandynamics/kafdrop:latest + networks: + - kafka-net + depends_on: + - broker1 + - broker2 + - broker3 + ports: + - 19000:9000 + environment: + KAFKA_BROKERCONNECT: broker1:9092,broker2:9092,broker3:9092 + restart: unless-stopped + +networks: + kafka-net: + driver: bridge diff --git a/redis/redis-sentinel/redis-sentinel.yml b/redis/redis-sentinel/redis-sentinel.yml deleted file mode 100644 index e69de29..0000000