zk-kafka-multiple-nodes-stack_prod.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
version: '3'
services:
  grafana:
    image: "grafana/grafana"
    ports:
     - "3000:3000"
    environment:
      GF_PATHS_DATA : /var/lib/grafana
      GF_SECURITY_ADMIN_PASSWORD : kafka
    volumes:
     - ./grafana/provisioning:/etc/grafana/provisioning
     - ./grafana/dashboards:/var/lib/grafana/dashboards
    container_name: grafana   
    networks:
     - paritex-kakfa
    depends_on:
     - prometheus

  prometheus:
    image: "prom/prometheus"
    ports:
     - "9090:9090"
    volumes:
     - ./etc/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
    command: "--config.file=/etc/prometheus/prometheus.yml"
    container_name: prometheus
    networks:
     - paritex-kakfa
   

  jmx-kafka101:
    image: "sscaling/jmx-prometheus-exporter"
    ports:
     - "5556:5556"
    environment:
     CONFIG_YML : "/etc/jmx_exporter/config.yml"
    volumes:
     - ./etc/jmx_exporter/config_kafka101.yml:/etc/jmx_exporter/config.yml
    container_name: jmx-kafka101
    networks:
     - paritex-kakfa   
    depends_on:
     - kafka1

  jmx-kafka102:
    image: "sscaling/jmx-prometheus-exporter"
    ports:
     - "5557:5556"
    environment:
     CONFIG_YML : "/etc/jmx_exporter/config.yml"
    volumes:
     - ./etc/jmx_exporter/config_kafka102.yml:/etc/jmx_exporter/config.yml
    container_name: jmx-kafka102   
    networks:
     - paritex-kakfa
    depends_on:
     - kafka2

  jmx-kafka103:
    image: "sscaling/jmx-prometheus-exporter"
    ports:
     - "5558:5556"
    environment:
     CONFIG_YML : "/etc/jmx_exporter/config.yml"
    volumes:
     - ./etc/jmx_exporter/config_kafka103.yml:/etc/jmx_exporter/config.yml
    container_name: jmx-kafka103   
    networks:
     - paritex-kakfa
    depends_on:
     - kafka3

  zoo1:
    image: confluentinc/cp-zookeeper:latest
    networks:
     - paritex-kakfa   
    environment:
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_SERVERS: zoo1:22888:23888;zoo2:32888:33888;zoo3:42888:43888   
    ports:
     - 2181:2181
    container_name: zookeeper1
    volumes:
      - ../zk-multiple-kafka-multiple/zoo1/data:/data
      - ../zk-multiple-kafka-multiple/zoo1/datalog:/datalog

  zoo2:
    image: confluentinc/cp-zookeeper:latest
    networks:
     - paritex-kakfa
    environment:
      ZOOKEEPER_SERVER_ID: 2
      ZOOKEEPER_CLIENT_PORT: 2182
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_SERVERS: zoo1:22888:23888;zoo2:32888:33888;zoo3:42888:43888    
    ports:
     - 2182:2182
    container_name: zookeeper2
    volumes:
      - ../zk-multiple-kafka-multiple/zoo2/data:/data
      - ../zk-multiple-kafka-multiple/zoo2/datalog:/datalog 

  zoo3:
    image: confluentinc/cp-zookeeper:latest
    networks:
     - paritex-kakfa
    environment:
      ZOOKEEPER_SERVER_ID: 3
      ZOOKEEPER_CLIENT_PORT: 2183
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_SERVERS: zoo1:22888:23888;zoo2:32888:33888;zoo3:42888:43888   
    ports:
     - 2183:2183
    container_name: zookeeper3
    volumes:
      - ../zk-multiple-kafka-multiple/zoo3/data:/data
      - ../zk-multiple-kafka-multiple/zoo3/datalog:/datalog    

  kafka1:
    image: confluentinc/cp-kafka:latest
    hostname: kafka1
    networks:
     - paritex-kakfa
    depends_on:
      - zoo1
      - zoo2
      - zoo3
    ports:
      - 9092:9092
      - 9992:9992
    container_name: kafka1   
    volumes:
      - ../zk-multiple-kafka-multiple/kafka1/data:/var/lib/kafka/data1
      - ../.env/server.properties:/etc/server.properties  
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_JMX_PORT: 9992
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2182,zoo3:2183
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:29092,PLAINTEXT_HOST://kafka1:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka1:29092
      CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2182,zoo3:2183
      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
      CONFLUENT_METRICS_ENABLE: 'false'

  kafka2:
    image: confluentinc/cp-kafka:latest
    hostname: kafka2
    networks:
     - paritex-kakfa   
    depends_on:
      - zoo1
      - zoo2
      - zoo3
    ports:
      - 9093:9093
      - 9993:9993
    container_name: kafka2
    volumes:
      - ../zk-multiple-kafka-multiple/kafka2/data:/var/lib/kafka/data1
      - ../.env/server.properties:/etc/server.properties
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_JMX_PORT: 9993
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2182,zoo3:2183
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka2:29093,PLAINTEXT_HOST://kafka2:9093
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka2:29093
      CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2182,zoo3:2183     
      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
      CONFLUENT_METRICS_ENABLE: 'false'

  kafka3:  
    image: confluentinc/cp-kafka:latest
    hostname: kafka3    
    networks:
     - paritex-kakfa
    depends_on:
      - zoo1
      - zoo2
      - zoo3
    ports:
      - 9094:9094
      - 9994:9994
    container_name: kafka3
    volumes:
      - ../zk-multiple-kafka-multiple/kafka3/data:/var/lib/kafka/data1
      - ../.env/server.properties:/etc/server.properties
    environment:
      KAFKA_BROKER_ID: 103
      KAFKA_JMX_PORT: 9994
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2182,zoo3:2183
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka3:29094,PLAINTEXT_HOST://kafka3:9094
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka3:29094
      CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2182,zoo3:2183
      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
      CONFLUENT_METRICS_ENABLE: 'false'  

  kafdrop:
    image: obsidiandynamics/kafdrop   
    restart: "no"
    container_name: kafdrop 
    networks:
     - paritex-kakfa  
    ports:
      - 9000:9000
    environment:
      KAFKA_BROKERCONNECT: "kafka1:29092,kafka2:29093,kafka3:29094"
      JVM_OPTS: "-Xms16M -Xmx48M -Xss180K -XX:-TieredCompilation -XX:+UseStringDeduplication -noverify"
    depends_on:
      - "kafka1"
      - "kafka2"
      - "kafka3"
networks:
  paritex-kakfa:
   driver: bridge