docker-compose.yml

version: "3.8"
services:
  #---------------------------- ZOOKEEPER ----------------------------#
  zookeeper:
    image: confluentinc/cp-zookeeper:5.4.1
    container_name: zookeeper
    networks:
      - internal
    environment:
      - ZOOKEEPER_CLIENT_PORT=2181
      - ZOOKEEPER_TICK_TIME=2000
  #-------------------------------------------------------------------#
  #------------------------------ KAFKA ------------------------------#
  kafka:
    image: confluentinc/cp-kafka:5.4.1
    container_name: kafka
    networks:
      - internal
    depends_on:
      - zookeeper
    ports:
      - 9092:9092
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100

  kafka-connect:
    image: confluentinc/cp-kafka-connect-base:5.4.1
    container_name: kafka-connect
    networks:
      - internal
    depends_on:
      - kafka
      - s3-storage
    ports:
      - 8083:8083
    environment:
      CONNECT_BOOTSTRAP_SERVERS: "kafka:29092"
      CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect"
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: kafka-connect
      CONNECT_CONNECTOR_CLIENT_CONFIG_OVERRIDE_POLICY: "All"
      CONNECT_CONFIG_STORAGE_TOPIC: _kafka-connect-configs
      CONNECT_OFFSET_STORAGE_TOPIC: _kafka-connect-offsets
      CONNECT_STATUS_STORAGE_TOPIC: _kafka-connect-status
      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
      CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components/,/connectors/"
    volumes:
      - $PWD/kafka-connect/connectors:/connectors
      - $PWD/kafka-connect/aws_credentials:/root/.aws/credentials
    # In the command section, $ are replaced with $$ to avoid the error 'Invalid interpolation format for "command" option'
    command:
      - bash
      - -c
      - |
        # Nasty hack for ECS
        echo '127.0.0.1 kafka-connect' >> /etc/hosts
        #
        echo "Installing connector plugins"
        confluent-hub install --no-prompt confluentinc/kafka-connect-s3:5.4.1
        #
        echo "Launching Kafka Connect worker"
        /etc/confluent/docker/run & 
        #
        echo "Waiting for Kafka Connect to start listening on localhost ⏳"
        while : ; do
          curl_status=$$(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors)
          echo -e $$(date) " Kafka Connect listener HTTP state: " $$curl_status " (waiting for 200)"
          if [ $$curl_status -eq 200 ] ; then
            break
          fi
          sleep 5 
        done

        bash /connectors/command.sh

        sleep infinity
  #-------------------------------------------------------------------#
  #----------------------------- AWS S3 ------------------------------#
  aws-s3:
    image: localstack/localstack:latest
    container_name: s3-storage
    networks:
      - internal
    environment:
      - SERVICES=s3
      - DEBUG=1
      - DEFAULT_REGION=us-east-1
      - AWS_ACCESS_KEY_ID=infodium
      - AWS_SECRET_ACCESS_KEY=infodium
    ports:
      - 4566:4566
    volumes:
      - "./aws/s3:/docker-entrypoint-initaws.d"
      #- "./volume/s3-storage/.localstack:/tmp/localstack"

  #-------------------------------------------------------------------#
  #------------------------- ELASSTICSEARCH --------------------------#
  elasticsearch:
    image: elasticsearch:7.12.1
    container_name: elasticsearch
    environment:
      - cluster.name=docker-cluster
      - xpack.security.enabled=false
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - discovery.type=single-node
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536
        hard: 65536
    #volumes:
      #- ./volume/elasticsearch:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
      - 9300:9300
    networks:
      - internal
  #-------------------------------------------------------------------#
  #------------------------------ KIBANA -----------------------------#
  kibana:
    container_name: kibana
    image: kibana:7.12.1
    environment:
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
    ports:
      - 5601:5601
    depends_on:
      - elasticsearch
    networks:
      - internal
  #-------------------------------------------------------------------#
  #------------------------------ MYSQL ------------------------------#
  mysql:
    image: mysql:latest
    container_name: mysql
    ports:
      - 3306:3306
    networks:
      - internal
    environment:
      - MYSQL_ROOT_PASSWORD=123456
  #-------------------------------------------------------------------#
  #------------------------- EVENT GENERATOR -------------------------#
  eventgen:
    build: ./eventgen
    image: eventgen
    container_name: eventgenerator
    networks:
      - internal
    depends_on:
      - mysql
      - aws-s3
      - kafka
      - elasticsearch
      - kibana
      - kafka-connect
    environment:
      - MYSQL_HOST=mysql
      - MYSQL_USER=root
      - MYSQL_PASS=123456
      - MYSQL_DATABASE=football
      - KAFKA_HOST=kafka
      - KAFKA_PORT=29092
      - KAFKA_TOPIC=football
      - DATA_DIR=/eventgen/resources/data
      - INPUT_FILE=events.csv
      - DELAY=2
  #-------------------------------------------------------------------#
  #------------------------------- FLINK -----------------------------#
  jobmanager:
    image: flink:1.13.0-scala_2.11
    container_name: "flink-jobmanager"
    depends_on: 
      - kafka
      - eventgen
      - mysql
    ports:
      - "8081:8081"
    networks: 
      - internal
    command: standalone-job --job-classname infodium.flink.StreamingJob
    volumes:
      - ./flink/job:/opt/flink/usrlib
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        parallelism.default: 1

  taskmanager:
    image: flink:1.13.0-scala_2.11
    container_name: "flink-taskmanager"
    depends_on:
      - jobmanager
    networks: 
      - internal
    command: taskmanager
    scale: 1
    volumes:
      - ./flink/job:/opt/flink/usrlib
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        taskmanager.numberOfTaskSlots: 4
        parallelism.default: 1
#-------------------------------------------------------------------#
#----------------------------- NETWORKS ----------------------------#
networks:
  external:
    name: external_network
    driver: bridge
    ipam:
      config:
        - subnet: 10.5.0.0/16
          gateway: 10.5.0.1
  internal:
    name: internal_network
    driver: bridge
    ipam:
      config:
        - subnet: 10.6.0.0/16
          gateway: 10.6.0.1
#-------------------------------------------------------------------#