Generate Random Avro data for Kafka Topic

datagen.sh

#!/bin/bash

if [ $# -eq 5 ]
then
uuid=$(uuidgen)
configTempFile=/tmp/$uuid
topicName=$1
schemaName=$2
schemaRegistryUrl=$3
iterations=$4
BOOTSTRAP_SERVERS=$5

curl -s $schemaRegistryUrl/subjects/$schemaName/versions/latest | jq -r '.schema' >> /tmp/$schemaName
cat <<EOF > $configTempFile
name=$uuid
connector.class=io.confluent.kafka.connect.datagen.DatagenConnector
tasks.max=1
kafka.topic=$topicName
schema.filename=/tmp/$schemaName
value.converter.schema.registry.url=$schemaRegistryUrl
key.converter=org.apache.kafka.connect.storage.StringConverter
value.converter=io.confluent.connect.avro.AvroConverter
value.converter.schemas.enable=true
max.interval=1
iterations=$iterations
EOF


echo "created config file $configTempFile"

sed -r -i "s/bootstrap.servers=localhost:9092/bootstrap.servers=$BOOTSTRAP_SERVERS/g" $CONFLUENT_HOME/etc/kafka/connect-standalone.properties

sed -r -i "s@plugin.path=share/java@plugin.path=$CONNECT_JAR_PATH@g" $CONFLUENT_HOME/etc/kafka/connect-standalone.properties

touch test.log

$CONFLUENT_HOME/bin/connect-standalone $CONFLUENT_HOME/etc/kafka/connect-standalone.properties $configTempFile > test.log &

echo "start tail"
tail -f test.log | while read LOGLINE
do
  echo $LOGLINE
  [[ "${LOGLINE}" == *"org.apache.kafka.connect.errors.ConnectException: Stopping connector: generated the configured"* ]] && exit 0
done
else
echo "You should mention 5 parameter exactly: topicName schemaName schemaRegistryUrl iterations bootstrapServers"
fi

Dockerfile

FROM openjdk:8-jdk

#INIT
RUN apt-get update && \
    apt-get install -y jq uuid-runtime unzip

ENV CONFLUENT_VERSION 4.1.3

ENV CONFLUENT_URL http://packages.confluent.io/archive/4.1/confluent-oss-4.1.3-2.11.tar.gz

ENV CONNECT_JAR_URL <REPLACE_DATAGENET : Get the jar from here https://www.confluent.io/connector/kafka-connect-datagen/>

ENV CONFLUENT_HOME /tmp/confluent-$CONFLUENT_VERSION

ENV CONNECT_JAR_PATH /tmp/connect-jar

WORKDIR /tmp

RUN curl $CONFLUENT_URL | tar xvz

RUN wget -O connect-jar.zip $CONNECT_JAR_URL && \
    unzip connect-jar.zip -d connect-jar && \
    chmod -R 777 $CONFLUENT_HOME/etc

COPY datagen.sh ./
RUN chmod +x datagen.sh

ENTRYPOINT ["./datagen.sh"]