Docker Compose를 사용하여 Kafka 셋업하기

코드는 여기 참고

1개의 Zookeeper와 3개의 Kafka broker를 갖는 Cluster

 version: '3.6'
 
services:
  zoo1:
    image: confluentinc/cp-zookeeper:6.2.1
    container_name: cluster-zoo1
    hostname: zoo1
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVERS: zoo1:2888:3888
 
 
  kafka1:
    image: confluentinc/cp-kafka:6.2.1
    container_name: cluster-kafka1
    hostname: kafka1
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
      KAFKA_BROKER_ID: 1
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
    depends_on:
      - zoo1
 
  kafka2:
    image: confluentinc/cp-kafka:6.2.1
    container_name: cluster-kafka2
    hostname: kafka2
    ports:
      - "9093:9093"
    environment:
      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
      KAFKA_BROKER_ID: 2
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
    depends_on:
      - zoo1
 
 
  kafka3:
    image: confluentinc/cp-kafka:6.2.1
    container_name: cluster-kafka3
    hostname: kafka3
    ports:
      - "9094:9094"
    environment:
      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19094,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
      KAFKA_BROKER_ID: 3
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
    depends_on:
      - zoo1
 
 
  kafka-ui:
    image: provectuslabs/kafka-ui
    container_name: cluster-kafka-ui
    ports:
      - "8080:8080"
    environment:
      KAFKA_CLUSTERS_0_NAME: local
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: PLAINTEXT://kafka1:19092,PLAINTEXT://kafka2:19093,PLAINTEXT://kafka3:19094
      KAFKA_CLUSTERS_0_ZOOKEEPER: "zoo1:2181"
    depends_on:
      - zoo1
      - kafka1
      - kafka2
      - kafka3

GUI management tool이 필요없는 경우 kafka-ui service는 삭제한다.

Kafka 관련 Stack

 version: '3.6'
 
services:
  zoo1:
    image: confluentinc/cp-zookeeper:6.2.1
    container_name: full-zoo1
    hostname: zoo1
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVERS: zoo1:2888:3888
 
  kafka1:
    image: confluentinc/cp-kafka:6.2.1
    container_name: full-kafka1
    hostname: kafka1
    ports:
      - "9092:9092"
      - "9999:9999"
    environment:
      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
      KAFKA_BROKER_ID: 1
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_JMX_PORT: 9999
      KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1}
    depends_on:
      - zoo1
 
  kafka-schema-registry:
    image: confluentinc/cp-schema-registry:6.2.1
    container_name: full-kafka-schema-registry
    hostname: kafka-schema-registry
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092
      SCHEMA_REGISTRY_HOST_NAME: kafka-schema-registry
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
    depends_on:
      - zoo1
      - kafka1
 
 
  kafka-rest-proxy:
    image: confluentinc/cp-kafka-rest:6.2.1
    container_name: full-kafka-rest-proxy
    hostname: kafka-rest-proxy
    ports:
      - "8082:8082"
    environment:
      # KAFKA_REST_ZOOKEEPER_CONNECT: zoo1:2181
      KAFKA_REST_LISTENERS: http://0.0.0.0:8082/
      KAFKA_REST_SCHEMA_REGISTRY_URL: http://kafka-schema-registry:8081/
      KAFKA_REST_HOST_NAME: kafka-rest-proxy
      KAFKA_REST_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092
    depends_on:
      - zoo1
      - kafka1
      - kafka-schema-registry
 
 
  kafka-connect:
    image: confluentinc/cp-kafka-connect:6.2.1
    container_name: full-kafka-connect
    hostname: kafka-connect
    ports:
      - "8083:8083"
    environment:
      CONNECT_BOOTSTRAP_SERVERS: "kafka1:19092"
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: compose-connect-group
      CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
      CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
      CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
      CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://kafka-schema-registry:8081'
      CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://kafka-schema-registry:8081'
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect"
      CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
      CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_PLUGIN_PATH: '/usr/share/java,/etc/kafka-connect/jars,/usr/share/confluent-hub-components'
    volumes:
      - ./connectors:/etc/kafka-connect/jars/
    depends_on:
      - zoo1
      - kafka1
      - kafka-schema-registry
      - kafka-rest-proxy
    command:
      - bash
      - -c
      - |
        confluent-hub install --no-prompt debezium/debezium-connector-mysql:latest
        confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:0.4.0
        /etc/confluent/docker/run
 
  ksqldb-server:
    image: confluentinc/cp-ksqldb-server:6.2.1
    container_name: full-ksqldb-server
    hostname: ksqldb-server
    ports:
      - "8088:8088"
    environment:
      KSQL_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092
      KSQL_LISTENERS: http://0.0.0.0:8088/
      KSQL_KSQL_SERVICE_ID: ksqldb-server_
    depends_on:
      - zoo1
      - kafka1
 
  zoonavigator:
    image: elkozmon/zoonavigator:0.8.0
    container_name: full-zoonavigator
    ports:
      - "8004:8000"
    environment:
      HTTP_PORT: 8000
      AUTO_CONNECT_CONNECTION_STRING: zoo1:2181
 
  kafka-ui:
    image: provectuslabs/kafka-ui
    container_name: full-kafka-ui
    ports:
      - "8080:8080"
    environment:
      KAFKA_CLUSTERS_0_NAME: local
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: PLAINTEXT://kafka1:19092
      KAFKA_CLUSTERS_0_ZOOKEEPER: "zoo1:2181"
      KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://kafka-schema-registry:8081
      KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: kafka-connect
      KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: http://kafka-connect:8083
      KAFKA_CLUSTERS_0_KSQLDBSERVER: http://ksqldb-server:8088
    depends_on:
      - zoo1
      - kafka1
      - kafka-schema-registry
      - kafka-connect
      - ksqldb-server

GUI management tool이 필요없는 경우 kafka-ui service는 삭제한다.

참고:

'Data Engineering > Kafka' 카테고리의 다른 글

Kafka Partition key를 사용하여 데이터 순서대로 처리하기 (0)	2023.01.12
Kafka의 정확히 한 번 전달(exactly-once delivery) (0)	2022.03.19

내 블로그 - 관리자 홈 전환	`Q` `Q`
새 글 쓰기	`W` `W`

글 수정 (권한 있는 경우)	`E` `E`
댓글 영역으로 이동	`C` `C`

이 페이지의 URL 복사	`S` `S`
맨 위로 이동	`T` `T`
티스토리 홈 이동	`H` `H`
단축키 안내	`Shift` + `/` `⇧` + `/`

Be an Overachiever

Docker Compose를 사용하여 Kafka 셋업하기

1개의 Zookeeper와 3개의 Kafka broker를 갖는 Cluster

Kafka 관련 Stack

'Data Engineering > Kafka' 카테고리의 다른 글

댓글

티스토리툴바

단축키

내 블로그

블로그 게시글

모든 영역

	version: '3.6'

	services:
	zoo1:
	image: confluentinc/cp-zookeeper:6.2.1
	container_name: cluster-zoo1
	hostname: zoo1
	ports:
	- "2181:2181"
	environment:
	ZOOKEEPER_CLIENT_PORT: 2181
	ZOOKEEPER_SERVER_ID: 1
	ZOOKEEPER_SERVERS: zoo1:2888:3888


	kafka1:
	image: confluentinc/cp-kafka:6.2.1
	container_name: cluster-kafka1
	hostname: kafka1
	ports:
	- "9092:9092"
	environment:
	KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
	KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
	KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
	KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
	KAFKA_BROKER_ID: 1
	KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
	depends_on:
	- zoo1

	kafka2:
	image: confluentinc/cp-kafka:6.2.1
	container_name: cluster-kafka2
	hostname: kafka2
	ports:
	- "9093:9093"
	environment:
	KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093
	KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
	KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
	KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
	KAFKA_BROKER_ID: 2
	KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
	depends_on:
	- zoo1


	kafka3:
	image: confluentinc/cp-kafka:6.2.1
	container_name: cluster-kafka3
	hostname: kafka3
	ports:
	- "9094:9094"
	environment:
	KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19094,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094
	KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
	KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
	KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
	KAFKA_BROKER_ID: 3
	KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
	depends_on:
	- zoo1


	kafka-ui:
	image: provectuslabs/kafka-ui
	container_name: cluster-kafka-ui
	ports:
	- "8080:8080"
	environment:
	KAFKA_CLUSTERS_0_NAME: local
	KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: PLAINTEXT://kafka1:19092,PLAINTEXT://kafka2:19093,PLAINTEXT://kafka3:19094
	KAFKA_CLUSTERS_0_ZOOKEEPER: "zoo1:2181"
	depends_on:
	- zoo1
	- kafka1
	- kafka2
	- kafka3

Docker Compose를 사용하여 Kafka 셋업하기

1개의 Zookeeper와 3개의 Kafka broker를 갖는 Cluster

Kafka 관련 Stack

'Data Engineering > Kafka' 카테고리의 다른 글

관련글

댓글

티스토리툴바

단축키

내 블로그

블로그 게시글

모든 영역