Run Kafka using Docker

run_kafka_using_docker.MD

Create a file with name docker-compose.yml

version: "3.7"
services:
  zookeeper:
    restart: always
    image: docker.io/bitnami/zookeeper:3.8
    ports:
      - "2181:2181"
    volumes:
      - "zookeeper-volume:/bitnami"
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes
  kafka:
    restart: always
    image: docker.io/bitnami/kafka:3.3
    ports:
      - "9093:9093"
    volumes:
      - "kafka-volume:/bitnami"
    environment:
      - KAFKA_BROKER_ID=1
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT
      - KAFKA_CFG_LISTENERS=CLIENT://:9092,EXTERNAL://:9093
      - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka:9092,EXTERNAL://localhost:9093
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT
    depends_on:
      - zookeeper
volumes:
  kafka-volume:
  zookeeper-volume:

Command to run kafka inside docker

docker-compose build #run this first time only
docker-compose up

Create a file to test if kafka working locally - name the file check_kafka.js

const { Kafka } = require('kafkajs');

// Define your Kafka broker(s)
const kafkaBrokers = ['localhost:9093'];

// Create a Kafka instance
const kafka = new Kafka({
  clientId: 'kafka-test-client',
  brokers: kafkaBrokers,
});

// Create a Kafka producer
const producer = kafka.producer();

// Create a Kafka consumer
const consumer = kafka.consumer({ groupId: 'kafka-test-group' });

// Function to produce a test message
async function produceMessage() {
  await producer.connect();
  await producer.send({
    topic: 'test-topic',
    messages: [{ value: 'Hello, Kafka!' }],
  });
  await producer.disconnect();
}

// Function to consume messages
async function consumeMessages() {
  await consumer.connect();
  await consumer.subscribe({ topic: 'test-topic', fromBeginning: true });

  await consumer.run({
    eachMessage: async ({ message }) => {
      console.log(`Received message: ${message.value}`);
    },
  });

  // Uncomment the following line if you want to stop the consumer after a certain period
  // setTimeout(async () => await consumer.disconnect(), 5000);
}

// Run the test
async function runTest() {
  try {
    await produceMessage();
    await consumeMessages();
  } catch (error) {
    console.error('Error:', error);
  } finally {
    await producer.disconnect();
  }
}

// Run the test
runTest();

connnect with kafka container (to check if it's runnig)

# to find containers with name
docker ps --format '{{.Names}}'

docker exec -it setup-kafka-kafka-1 bin/bash #setup-kafka-kafka-1 is my kafka container name

# find kafka ip
docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' <container_id_or_name>

# to check if kafka working in machine
node check_kafka.js

On running kafka using docker you will get output something like this

If kafka working fine then you will get output something like this