Startup scripts for Zookeeper and Kafka with self discovery in AWS

start-kafka-node.sh

#!/usr/bin/env bash

ENVIRONMENT="{{ target_env }}" # Ansible will replace this one
AWS_SERVICE_LOG_GROUP="{{ kafka_log_group_name }}" # Ansible will replace this one (automatically discovered)
AWS_SERVICE_LOG_STREAM=container-logs
DOCKER_CONTAINER="{{ kafka_docker_container }}" # Ansible will replace this one

# Get meta information about instance and its placement
AVAILABILITY_ZONE=`curl http://169.254.169.254/latest/meta-data/placement/availability-zone`
LOCAL_IP=`curl -s http://169.254.169.254/latest/meta-data/local-ipv4`
REGION="`echo \"$AVAILABILITY_ZONE\" | sed -e 's:\([0-9][0-9]*\)[a-z]*\$:\\1:'`"
INSTANCE_ID=`curl http://169.254.169.254/latest/meta-data/instance-id`

# This will give us the value of KafkaID tag that we will use as a unique ID for the node
BROKER_ID="`aws ec2 describe-tags \
  --region $REGION \
  --filters "Name=resource-id,Values=$INSTANCE_ID" "Name=key,Values=KafkaID" \
  --output=text | cut -f5`"

# This command searches for running zookeeper instances for the current environment
# and produces the output like:
#
# 10.23.91.156
# 1
# 10.23.91.176
# 2
# ...
#
# where odd lines are private IP addresses for instances
# and even lines are Zookeeper ID tag values
INSTANCES=$(aws ec2 describe-instances \
  --region=$REGION \
  --filters "Name=tag:Environment,Values=$ENVIRONMENT" "Name=tag:Role,Values=zookeeper" "Name=instance-state-name,Values=running" \
  --query 'Reservations[*].Instances[*].[PrivateIpAddress][]' \
  --output text | tr " " "\n")

# Iterates of the info we just retrieved and creates Zookeeper connection string like
# server.ID=IP:2888:3888 server.ID=IP:2888:3888 ...
mapfile -t INSTANCES <<< "$INSTANCES"
ZOO_SERVERS_STRING=""
for i in "${!INSTANCES[@]}"
do
  ZOO_SERVERS_STRING="${ZOO_SERVERS_STRING}${INSTANCES[i]}:2181,"
done

# Reset configuration file
cp /opt/kafka/server.properties.base /opt/kafka/server.properties
printf "\nzookeeper.connect="$ZOO_SERVERS_STRING >> /opt/kafka/server.properties
printf "\nbroker.id="$BROKER_ID >> /opt/kafka/server.properties
printf "\nbroker.rack="$AVAILABILITY_ZONE >> /opt/kafka/server.properties
printf "\nadvertised.listeners=PLAINTEXT://"$LOCAL_IP":9092\n" >> /opt/kafka/server.properties

# DEBUG environment configuration
LOG="Launching Kafka broker node with id=${BROKER_ID} and connection string=${ZOO_SERVERS_STRING}"
echo $LOG
echo $LOG >> /var/log/kafka-launch-script.log

# Remove all the existing containers
sudo docker stop $(docker ps -aq)
sudo docker rm $(docker ps -aq)

# Start a new docker container
sudo docker run \
  -d \
  -p 9092:9092 \
  -p 7203:7203 \
  --restart on-failure \
  -v /var/run/docker.sock:/var/run/docker.sock \
  -v /opt/kafka:/kafka \
  -v /opt/kafka/server.properties:/opt/kafka/config/server.properties \
  -e KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.rmi.port=7203 -Djava.rmi.server.hostname=$LOCAL_IP -Dcom.sun.management.jmxremote.port=7203" \
  -e JMX_PORT=7203 \
  --log-driver=awslogs \
  --log-opt awslogs-region="$REGION" \
  --log-opt awslogs-group="$AWS_SERVICE_LOG_GROUP" \
  --log-opt awslogs-stream="$AWS_SERVICE_LOG_STREAM" \
  $DOCKER_CONTAINER || error_exit 'Failed to launch Docker container'

# Optional: Web UI to make you happy
docker run \
  -d \
  -p 9000:9000 \
  --restart on-failure \
  -e ZK_HOSTS=$ZOO_SERVERS_STRING \
   sheepkiller/kafka-manager

zookeeper-start-node.sh

#!/usr/bin/env bash

# This scripts looks for the EC2 instances marked by tags Environment=$1 and Role=zookeeper
# to generate a connection string for Zookeeper using value from tag ZookeeperID

# For example if you have 3 instances in Environment=uat tagged by Role=zookeeper and ZookeeperID=1, 2 and 3
# then this script will gets connection string like "server.1=10.12.24.41:2888:3888 server.2=10.12.24.52:2888:3888 ..."

# Than official docker container will be started using those parameters

ENVIRONMENT="{{ target_env }}" # Ansible will replace this one
AWS_SERVICE_LOG_GROUP="{{ zookeeper_log_group_name }}" # Ansible will replace this one (automatically discovered)
AWS_SERVICE_LOG_STREAM=container-logs
DOCKER_CONTAINER="{{ zookeeper_docker_container }}"  # Ansible will replace this one

# Get meta information about instance and its placement
AVAILABILITY_ZONE=`curl http://169.254.169.254/latest/meta-data/placement/availability-zone`
REGION="`echo \"$AVAILABILITY_ZONE\" | sed -e 's:\([0-9][0-9]*\)[a-z]*\$:\\1:'`"
INSTANCE_ID=`curl http://169.254.169.254/latest/meta-data/instance-id`

# This will give us the value of Zookeeper ID tag that we will use as a unique ID for the node
ZOO_MY_ID="`aws ec2 describe-tags \
  --region $REGION \
  --filters "Name=resource-id,Values=$INSTANCE_ID" "Name=key,Values=ZookeeperID" \
  --output=text | cut -f5`"

# This command searches for running zookeeper instances for the current environment
# and produces the output like:
#
# 10.23.91.156
# 1
# 10.23.91.176
# 2
# ...
#
# where odd lines are private IP addresses for instances
# and even lines are Zookeeper ID tag values
INSTANCES=$(aws ec2 describe-instances \
  --region=$REGION \
  --filters "Name=tag:Environment,Values=$ENVIRONMENT" "Name=tag:Role,Values=zookeeper" "Name=instance-state-name,Values=running" \
  --query 'Reservations[*].Instances[*].[Tags[?Key==`ZookeeperID`].Value, PrivateIpAddress][]' \
  --output text | tr " " "\n")

# Iterates of the info we just retrieved and creates Zookeeper connection string like
# server.ID=IP:2888:3888 server.ID=IP:2888:3888 ...
mapfile -t INSTANCES <<< "$INSTANCES"
ZOO_SERVERS_STRING=""
for i in "${!INSTANCES[@]}"
do
    if [ $((i%2)) -eq 0 ];
    then
        ZOO_SERVERS_STRING="${ZOO_SERVERS_STRING}server.${INSTANCES[i+1]}=${INSTANCES[i]}:2888:3888 "
    fi
done

# DEBUG environment configuration
LOG="Launching Zookeeper node with id=${ZOO_MY_ID} and connection string=${ZOO_SERVERS_STRING}"
echo $LOG
echo $LOG >> /var/log/zookeeper-launch-script.log

# Remove all the existing containers
sudo docker stop $(docker ps -aq)
sudo docker rm $(docker ps -aq)

# Start a new docker container
sudo docker run \
  -d \
  --network=host \
  --restart on-failure \
  -e ZOO_MY_ID="$ZOO_MY_ID" \
  -e ZOO_SERVERS="$ZOO_SERVERS_STRING" \
  -v /opt/zookeeper:/data \
  --log-driver=awslogs \
  --log-opt awslogs-region="$REGION" \
  --log-opt awslogs-group="$AWS_SERVICE_LOG_GROUP" \
  --log-opt awslogs-stream="$AWS_SERVICE_LOG_STREAM" \
  $DOCKER_CONTAINER || error_exit 'Failed to launch Docker container'