extract.sh

sh
ZOOKEEPER_NODES=$(grep "zookeeper.connect=" /etc/kafka/server.properties | cut -d '=' -f 2)

zk_nodes=$(echo $ZOOKEEPER_NODES | tr "," "\n")
for zk_node in $zk_nodes
do
   zk_host=$(echo $zk_node | cut -d':' -f1)
   zk_port=$(echo $zk_node | cut -d':' -f2)
   zk_node_type=$(echo 'mntr' | nc $zk_host $zk_port | grep 'zk_server_state'| cut -f2)
   echo "$zk_node is $zk_node_type"

   if [[ "$zk_node_type" == "leader" ]]; then
     number_followers=$(echo 'mntr' | nc $zk_host $zk_port | grep 'zk_followers'| cut -f2)
     echo "Number of followers is : $number_followers"
   fi
done

echo "number of brokers setup in this zookeeper"
zookeeper-shell zookeeper:2181  ls /brokers/ids | grep "\["

BOOTSTRAP_SERVERS=localhost:9092
#if security is used
#CONSUMER_CONFIG="--command-config /etc/kafka/consumer.properties"
#if no security is used
CONSUMER_CONFIG=""

ech "Let us review all the properties to gather info"
echo ZOOKEEPER_NODES=$ZOOKEEPER_NODES
echo BOOTSTRAP_SERVERS=$BOOTSTRAP_SERVERS
echo CONSUMER_CONFIG=$CONSUMER_CONFIG


Echo "Let us gather all the consumer groups"
consumergroups="/tmp/consumer-groups-$RANDOM"
kafka-consumer-groups  --bootstrap-server $BOOTSTRAP_SERVERS $CONSUMER_CONFIG --list > $consumergroups
cat $consumergroups

echo "# Connect"
echo "~Please aware that the consumers groups will only be available if sinks are defined~"
echo "~When ip are discovered, that means there is a worker there, let's go on each of these ip to see how many workers are running~"
for group in $( cat $consumergroups | grep 'connect-'); do
   echo "## Connect $group"
   kafka-consumer-groups  --bootstrap-server $BOOTSTRAP_SERVERS $CONSUMER_CONFIG --describe --group $group | grep $group | awk '{print $8}'| cut -d '/' -f 2 | uniq -c
done

echo "# KSQL"
for group in $( cat $consumergroups | grep '_confluent-ksql'); do
   echo "## KSQL $group"
   kafka-consumer-groups --bootstrap-server $BOOTSTRAP_SERVERS $CONSUMER_CONFIG --describe --group $group | grep $group | awk '{print $8}'| cut -d '/' -f 2 | uniq -c
done

echo "# Control Center application ips"
for group in $( cat $consumergroups | grep '_confluent-controlcenter'); do
   echo "## Control Center $group"
   kafka-consumer-groups --bootstrap-server $BOOTSTRAP_SERVERS $CONSUMER_CONFIG --describe --group $group | grep $group | awk '{print $8}'| cut -d '/' -f 2 | uniq -c
done

echo "# Kafka Stream application ips"
for group in $( cat $consumergroups | grep -v 'connect-' | grep -v '_confluent-controlcenter' | grep -v '_confluent-ksql'); do
   echo "## Kafka Stream $group"
   kafka-consumer-groups --bootstrap-server $BOOTSTRAP_SERVERS $CONSUMER_CONFIG --describe --group $group | grep $group | grep StreamThread | awk '{print $8}'| cut -d '/' -f 2 | uniq -c
done

echo "# On each ip we need to run "
echo "ps -ef "
echo "ps -ef | grep -P '(confluent|kafka)' | grep -v grep"