kafka-connect-iothub.sh

#!/bin/bash

# Script is idempotent
CLUSTERNAME=$1
PASSWORD=$2
TOPIC=$3

# Import HDInsight helper method module
wget -O /tmp/HDInsightUtilities-v01.sh -q https://hdiconfigactions.blob.core.windows.net/linuxconfigactionmodulev01/HDInsightUtilities-v01.sh && source /tmp/HDInsightUtilities-v01.sh && rm -f /tmp/HDInsightUtilities-v01.sh

KAFKA_HOME=/usr/hdp/current/kafka-broker

# Download Kafka Connect for IotHub and place it in Kafka broker libs
# TODO: REPLACE URL WHEN THIS ISSUE IS RESOLVED!!! https://github.com/Azure/toketi-kafka-connect-iothub/issues/10
download_file https://hassaan.blob.core.windows.net/iothub/kafka-connect-iothub-assembly_2.11-0.6.jar $KAFKA_HOME/libs/kafka-connect-iothub-assembly_2.11-0.6.jar true

# Only the above jar file needs to be on every node
if [ `test_is_headnode` == 0 ]; then
  echo "Kafka Connect only needs to be configured on headnode, exiting ..."
  exit 0
fi

# Retrieve FQDN of Kafka brokers and Zookeeper using Ambari REST API
sudo apt install jq -y #jq is used below to parse response
KAFKAZKHOSTS=`curl -sS -u admin:$PASSWORD -G https://$CLUSTERNAME.azurehdinsight.net/api/v1/clusters/$CLUSTERNAME/services/ZOOKEEPER/components/ZOOKEEPER_SERVER | jq -r '["\(.host_components[].HostRoles.host_name):2181"] | join(",")' | cut -d',' -f1,2`
KAFKABROKERS=`curl -sS -u admin:$PASSWORD -G https://$CLUSTERNAME.azurehdinsight.net/api/v1/clusters/$CLUSTERNAME/services/KAFKA/components/KAFKA_BROKER | jq -r '["\(.host_components[].HostRoles.host_name):9092"] | join(",")' | cut -d',' -f1,2`
echo '$KAFKAZKHOSTS='$KAFKAZKHOSTS
echo '$KAFKABROKERS='$KAFKABROKERS

# Create our topic
$KAFKA_HOME/bin/kafka-topics.sh --create --replication-factor 3 --partitions 8 --topic $TOPIC --zookeeper $KAFKAZKHOSTS

# Create default Kafka Connect topics
$KAFKA_HOME/bin/kafka-topics.sh --create --replication-factor 3 --partitions 8 --topic connect-offsets --zookeeper $KAFKAZKHOSTS
$KAFKA_HOME/bin/kafka-topics.sh --create --replication-factor 3 --partitions 8 --topic connect-status --zookeeper $KAFKAZKHOSTS

# 'connect-configs' MUST HAVE SINGLE PARTITION
$KAFKA_HOME/bin/kafka-topics.sh --create --replication-factor 3 --partitions 1 --topic connect-configs --zookeeper $KAFKAZKHOSTS

# List topics
$KAFKA_HOME/bin/kafka-topics.sh --list --zookeeper $KAFKAZKHOSTS

# Use | as delimiter due to URLs in values, keep backup of properties file
sudo sed -i.bak "s|\(bootstrap\.servers=\).*\$|\1$KAFKABROKERS|" $KAFKA_HOME/conf/connect-distributed.properties

# Run Kafka Connect server
$KAFKA_HOME/bin/connect-distributed.sh -daemon $KAFKA_HOME/conf/connect-distributed.properties