docker-compose.yml

version: '3' 
services:

  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8
    container_name: namenode
    volumes:
      - /tmp/hdfs/namenode:/hadoop/dfs/name
    environment:
      - CLUSTER_NAME=test
    env_file:
      - ./hadoop-hive.env
    ports:
      - "50070:50070"
    networks:
      net_pet:
        ipv4_address: 172.27.1.5
  
  datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
    container_name: datanode
    volumes:
      - /tmp/hdfs/datanode:/hadoop/dfs/data
      - ./bank:/bank
    env_file:
      - ./hadoop-hive.env
    environment:
      SERVICE_PRECONDITION: "namenode:50070"
    depends_on:
      - namenode
    ports:
      - "50075:50075"
    networks:
      net_pet:
        ipv4_address: 172.27.1.6
  
  hive-server:
    image: bde2020/hive:2.3.2-postgresql-metastore
    container_name: hive-server
    env_file:
      - ./hadoop-hive.env
    environment:
      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
      SERVICE_PRECONDITION: "hive-metastore:9083"
    ports:
      - "10000:10000"
    depends_on:
      - hive-metastore
    networks:
      net_pet:
        ipv4_address: 172.27.1.7
  
  hive-metastore:
    image: bde2020/hive:2.3.2-postgresql-metastore
    container_name: hive-metastore
    env_file:
      - ./hadoop-hive.env
    command: /opt/hive/bin/hive --service metastore
    environment:
      SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432"
    ports:
      - "9083:9083"
    depends_on:
      - hive-metastore-postgresql
    networks:
      net_pet:
        ipv4_address: 172.27.1.8
  
  hive-metastore-postgresql:
    image: bde2020/hive-metastore-postgresql:2.3.0
    container_name: hive-metastore-postgresql
    depends_on:
      - datanode
    networks:
      net_pet:
        ipv4_address: 172.27.1.9
    
  spark-master:
    image: bde2020/spark-master:2.4.0-hadoop2.7
    container_name: spark-master
    ports:
      - 8080:8080
      - 7077:7077
    environment:
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
    env_file:
      - ./hadoop-hive.env
    networks:
      net_pet:
        ipv4_address: 172.27.1.10
     
  spark-worker:
    image: bde2020/spark-worker:2.4.0-hadoop2.7
    container_name: spark-worker
    depends_on:
      - spark-master
    environment:
      - SPARK_MASTER=spark://spark-master:7077
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
      - HIVE_CORE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore/metastore
    depends_on:
      - spark-master
    ports:
      - 8081:8081
    env_file:
      - ./hadoop-hive.env
    networks:
      net_pet:
        ipv4_address: 172.27.1.11

  zeppelin:
    image: apache/zeppelin:0.8.0
    container_name: zeppelin
    ports:
      - 19090:8080
    networks:
      net_pet:
        ipv4_address: 172.27.1.12

  hue:
      image: gethue/hue:20191107-135001
      hostname: hue
      container_name: hue
      dns: 8.8.8.8
      ports:
      - "8888:8888"
      volumes:
        - ./hue-overrides.ini:/usr/share/hue/desktop/conf/z-hue.ini
      depends_on:
      - "database"
      networks:
        net_pet:
          ipv4_address: 172.27.1.13
  
  database:
      image: mysql:5.7
      container_name: database
      ports:
          - "33061:3306"
      command: --init-file /data/application/init.sql
      volumes:
          - /tmp/mysql/data:/var/lib/mysql
          - ./init.sql:/data/application/init.sql
      environment:
          MYSQL_ROOT_USER: hueroot
          MYSQL_ROOT_PASSWORD: secret
          MYSQL_DATABASE: hue
          MYSQL_PASSWORD: secret
          MYSQL_USER: hueroot
      networks:
        net_pet:
          ipv4_address: 172.27.1.14

  zookeeper:
    hostname: zookeeper
    container_name: zookeeper
    image: bitnami/zookeeper:latest
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes
    ports:
      - "2181:2181"
    networks:
      net_pet:
        ipv4_address: 172.27.1.15

  kafka:
    image: landoop/fast-data-dev:latest
    container_name: kafka
    ports:
      - "9092:9092"
      - "3030:3030"    
    environment:
      KAFKA_ADVERTISED_HOST_NAME: 172.27.1.16
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
    networks:
      net_pet:
        ipv4_address: 172.27.1.16  

  streamsets:
    image: streamsets/datacollector:3.13.0-latest
    container_name: streamsets
    ports:
      - "18630:18630"
    networks:
      net_pet:
        ipv4_address: 172.27.1.17  

  nifi:
    image: apache/nifi:latest
    container_name: nifi
    ports:
      - 9999:9999
    environment:
      - NIFI_WEB_HTTP_PORT=9999
      - NIFI_CLUSTER_IS_NODE=true
      - NIFI_CLUSTER_NODE_PROTOCOL_PORT=8082
      - NIFI_ZK_CONNECT_STRING=zookeeper:2181
      - NIFI_ELECTION_MAX_WAIT=1 min 
    networks:
      net_pet:
        ipv4_address: 172.27.1.18    

networks:
  net_pet:
    ipam:
      driver: default
      config:
        - subnet: 172.27.0.0/16