Apache Kafka pub/sub basic example in MSK

add-topic.sh

#!/bin/bash

./kafka_2.12-2.3.1/bin/kafka-topics.sh --create --topic foobar --bootstrap-server b-1.demo-cluster-1.8hn95l.c3.kafka.eu-west-1.amazonaws.com:9092 --partitions 4 --replication-factor 2

consumer.py

from kafka import KafkaConsumer

consumer = KafkaConsumer('foobar', bootstrap_servers=['b-1.demo-cluster-1.8hn95l.c3.kafka.eu-west-1.amazonaws.com:9094','b-2.demo-cluster-1.8hn95l.c3.kafka.eu-west-1.amazonaws.com:9094'], security_protocol="SSL")

for msg in consumer:
    print(msg)

producer.py

from kafka import KafkaProducer
from datetime import datetime
import time

producer = KafkaProducer(bootstrap_servers=['b-1.demo-cluster-1.8hn95l.c3.kafka.eu-west-1.amazonaws.com:9094','b-2.demo-cluster-1.8hn95l.c3.kafka.eu-west-1.amazonaws.com:9094'], security_protocol="SSL")


while True:
    now = datetime.now()
    msg = str.encode("random event at {}".format(now.strftime("%m/%d/%Y, %H:%M:%S")))
    producer.send('foobar', msg)
    print("Sent message at {}".format(now.strftime("%m/%d/%Y, %H:%M:%S")))
    time.sleep(3)