Ejemplo introducción a Kafka - christian-ariza.net

01-Producer.ipynb

02-consumer.ipynb

03-spark.ipynb

docker-compose.yml

version: '3'
services:
  kafka:
    image: confluentinc/cp-kafka
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
    depends_on:
      - zookeeper

  zookeeper:
    image: confluentinc/cp-zookeeper
    environment:
      - ZOOKEEPER_CLIENT_PORT=2181
    ports:
      - "2181:2181"
  spark-executor:
    image: spark:3.5.1-scala2.12-java11-ubuntu
    environment:
      SPARK_MASTER_HOST: spark-master
      SPARK_MASTER_PORT: 7077
    command: spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
    volumes:
      - ./shared_folder:/workspace/data
    deploy:
      replicas: 2
    depends_on:
      - spark-master
  spark-master:
    image: spark:3.5.1-scala2.12-java11-ubuntu
    environment:
      SPARK_MASTER_HOST: spark-master
      SPARK_MASTER_PORT: 7077
      SPARK_LOCAL_IP: spark-master
    command: spark-class org.apache.spark.deploy.master.Master
    volumes:
      - ./shared_folder:/workspace/data
  spark-notebook:
    build:
      context: .
      dockerfile_inline: |
        FROM quay.io/jupyter/pyspark-notebook:spark-3.5.1
        #set password for jupyter
        RUN echo '{"IdentityProvider": {"hashed_password": "argon2:$$argon2id$$v=19$$m=10240,t=10,p=8$$zUYc32oQmbROa0YxSdntdw$$5hOMyxMMdml9/pM1Jc8A1GNMhi1d3cEEZiBW3KjJhCY"}}' >> /home/jovyan/.jupyter/jupyter_server_config.json && \
        chmod 600 /home/jovyan/.jupyter/jupyter_server_config.json
    ports:
      - "4040:4040"
      - "8888:8888"
      - "38889:38889"
      - "7777:7777"
    volumes:
      - ./:/home/jovyan/work
  minio-server:
    image: quay.io/minio/minio
    command: server /data --console-address ":9001"
    ports:
      - "9000:9000"
      - "9001:9001"
    environment:
      MINIO_ROOT_USER: minio
      MINIO_ROOT_PASSWORD: minio2024