docker-compose. with Apache Spark Cluster, PostgreSQL and Apache Livy server

docker-compose.yaml

version: '3'

services:
  spark-master:
    image: docker.io/bitnami/spark:2
    environment:
      - SPARK_MODE=master
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
    user: root
    volumes:
      - type: bind
        source: ./spark/conf/log4j.properties
        target: /opt/bitnami/spark/conf/log4j.properties
      - ./spark/spark-defaults.conf:/opt/bitnami/spark/conf/spark-defaults.conf
      - ./spark/jars_dir:/opt/bitnami/spark/ivy:z
    ports:
      - '8080:8080'
      - '7077:7077'
    networks:
      - net

  spark-worker1:
    image: docker.io/bitnami/spark:2
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077
      - SPARK_WORKER_MEMORY=1G
      - SPARK_WORKER_CORES=1
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
    user: root
    volumes:
      - type: bind
        source: ./spark/conf/log4j.properties
        target: /opt/bitnami/spark/conf/log4j.properties
      - ./spark/spark-defaults.conf:/opt/bitnami/spark/conf/spark-defaults.conf
      - ./spark/jars_dir:/opt/bitnami/spark/ivy:z
    ports:
      - '8081:8081'   
    networks:
      - net
    depends_on:
      - spark-master

  spark-worker2:
    image: docker.io/bitnami/spark:2
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077
      - SPARK_WORKER_MEMORY=1G
      - SPARK_WORKER_CORES=1
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
    user: root
    volumes:
      - type: bind
        source: ./spark/conf/log4j.properties
        target: /opt/bitnami/spark/conf/log4j.properties
      - ./spark/spark-defaults.conf:/opt/bitnami/spark/conf/spark-defaults.conf
      - ./spark/jars_dir:/opt/bitnami/spark/ivy:z
    ports:
      - '8082:8082'   
    networks:
      - net
    depends_on:
      - spark-master

  livy-server:
    container_name: livy_server
    build: ./livy-server/
    command: ["sh", "-c", "/opt/bitnami/livy/bin/livy-server"]
    user: root
    volumes:
      - type: bind
        source: ./livy-server/conf/
        target: /opt/bitnami/livy/conf/
      - type: bind
        source: ./livy-server/target/
        target: /target/
      - type: bind
        source: ./livy-server/data/
        target: /data/
    ports:
      - '8998:8998'
    networks:
      - net
    depends_on:
      - spark-master
      - spark-worker1
      - spark-worker2

  db:
    container_name: pg_container
    image: postgres
    restart: always
    environment:
        POSTGRES_USER: "postgres"
        POSTGRES_PASSWORD: "12345"
        POSTGRES_DB: "db"
        POSTGRES_HOST_AUTH_METHOD: trust
    ports:
      - "5432:5432"
    networks:
      - net
        
volumes:
  pg_data:

networks:
  net:
    driver: bridge