easy listener for heroku kafka

kafka-consumer.sh

#!/usr/bin/env bash
set -e

## Prereqs:
## brew install heroku
## brew install kafkacat
## heroku login -i

## Usage exmaple
## $ APP=roadie-admin-qa ./kafka-listener.sh gig.events
##     APP defaults to roadie-api-qa
##     TOPIC defaults to beta.events

# Get config values/creds from heroku by env var name
#
function get_config_val() {
  config_key="$1"

  heroku config:get "$config_key" -a "$APP"
}

APP="${APP:-roadie-api-qa}"
TOPIC="${1:-beta.events}"
KAFKA_PREFIX="$(get_config_val KAFKA_PREFIX)"
KAFKA_URL="$(get_config_val KAFKA_URL | cut -f 1 -d ',' | sed 's#kafka\+ssl\://##g')"
CONSUMER_GROUP="${CONSUMER_GROUP:-cli-consumer-$RANDOM}"

kafkacat -C -t "${KAFKA_PREFIX}${TOPIC}" \
         -b "$KAFKA_URL" \
         -X group.id="$CONSUMER_GROUP" \
         -X security.protocol=ssl \
         -X ssl.key.location=<(get_config_val KAFKA_CLIENT_CERT_KEY) \
         -X ssl.ca.location=<(get_config_val KAFKA_TRUSTED_CERT) \
         -X ssl.certificate.location=<(get_config_val KAFKA_CLIENT_CERT)

#         -o beginning \

kafka-producer.sh

#!/usr/bin/env bash
set -e

## Prereqs:
## brew install heroku
## brew install kafkacat
## heroku login -i

## Usage exmaple
## $ APP=roadie-admin-qa ./kafka-writer.sh gig.events
##     APP defaults to roadie-api-qa
if [ -z "$1" ]; then
  echo "You must specify a topic as the first argument"
  exit 250
fi

TOPIC="$1"
APP="${APP:-roadie-api-qa}"

function get_config_val() {
  config_key="$1"

  heroku config:get "$config_key" -a "$APP"
}

APP="${APP:-roadie-api-qa}"
TOPIC="${1:-beta.events}"
KAFKA_PREFIX="$(get_config_val KAFKA_PREFIX)"
KAFKA_URL="$(get_config_val KAFKA_URL | cut -f 1 -d ',' | sed 's#kafka\+ssl\://##g')"

kafkacat -P -p -1 \
         -t "${KAFKA_PREFIX}${TOPIC}" \
         -b "$KAFKA_URL" \
         -X security.protocol=ssl \
         -X ssl.key.location=<(get_config_val KAFKA_CLIENT_CERT_KEY) \
         -X ssl.ca.location=<(get_config_val KAFKA_TRUSTED_CERT) \
         -X ssl.certificate.location=<(get_config_val KAFKA_CLIENT_CERT) \
         -l <(echo "{\"created_at\": \"$(date)\"}")