Mirror a https://github.com/Yolean/kubernetes-kafka cluster to local

docker-compose.yml

version: "2"
services:
  mirror-kafka:
    image: solsson/kafka:1.0.1@sha256:1a4689d49d6274ac59b9b740f51b0408e1c90a9b66d16ad114ee9f7193bab111
    environment:
    - SOURCE_BOOTSTRAP=kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
    - TARGET_BOOTSTRAP=kafka:9092
    - WHITELIST=$MIRROR_TOPIC_WHITELIST
    links:
    - kafka
    - kafka-forward-0:kafka-0.broker.kafka.svc.cluster.local
    - kafka-forward-1:kafka-1.broker.kafka.svc.cluster.local
    - kafka-forward-2:kafka-2.broker.kafka.svc.cluster.local
    entrypoint:
    - bash
    - -cex
    - >
      echo "bootstrap.servers=$$SOURCE_BOOTSTRAP" > ./config/consumer.properties;
      echo "group.id=dev-mirror-$USER"           >> ./config/consumer.properties;
      echo "auto.offset.reset=none"              >> ./config/consumer.properties;
      echo "bootstrap.servers=$$TARGET_BOOTSTRAP" > ./config/producer.properties;
      sed -i 's/WARN/INFO/g' ./config/tools-log4j.properties;
      echo "log4j.logger.org.apache.kafka.clients.Metadata=DEBUG" >> ./config/tools-log4j.properties;
      echo "log4j.logger.org.apache.kafka.clients.consumer.internals.AbstractCoordinator=DEBUG" >> ./config/tools-log4j.properties;
      echo "log4j.logger.org.apache.kafka.clients.NetworkClient=DEBUG" >> ./config/tools-log4j.properties;
      ./bin/kafka-mirror-maker.sh --consumer.config ./config/consumer.properties --producer.config ./config/producer.properties --whitelist $$WHITELIST;

  kafka-forward-0:
    build: ./mirror-kafka
    volumes:
    - $MIRROR_SOURCE_KUBECONFIG:/root/.kube/config
    environment:
    - FORWARD_NAMESPACE=kafka
    - FORWARD_POD=kafka-0
    expose:
    - "9092"

  kafka-forward-1:
    build: ./mirror-kafka
    volumes:
    - $MIRROR_SOURCE_KUBECONFIG:/root/.kube/config
    environment:
    - FORWARD_NAMESPACE=kafka
    - FORWARD_POD=kafka-1
    expose:
    - "9092"

  kafka-forward-2:
    build: ./mirror-kafka
    volumes:
    - $MIRROR_SOURCE_KUBECONFIG:/root/.kube/config
    environment:
    - FORWARD_NAMESPACE=kafka
    - FORWARD_POD=kafka-2
    expose:
    - "9092"

### The docker image for ./mirror-kafka
### pending https://github.com/kubernetes/kubernetes/pull/46517

FROM solsson/kubectl-kafkacat@sha256:01f72d7e6913c2652c49f654e359f627b4b42a7c5e008a5411ed58655eb9c939

RUN set -ex; \
  export DEBIAN_FRONTEND=noninteractive; \
  runDeps='rinetd net-tools'; \
  buildDeps=''; \
  apt-get update && apt-get install -y $runDeps $buildDeps --no-install-recommends; \
  rm -rf /var/lib/apt/lists/*; \
  \
  apt-get purge -y --auto-remove $buildDeps; \
  rm /var/log/dpkg.log /var/log/apt/*.log

ENV \
  FORWARD_NAMESPACE=default \
  FORWARD_POD=you-need-to-set-a-pod-name \
  FORWARD_PORT=9092:9092
# 9092 is the default because we wrote this for kafka forwarding

COPY port-forward.sh /usr/local/bin/kubectl-port-forward

ENTRYPOINT [ "kubectl-port-forward" ]

### port-forward.sh

#!/bin/sh
set -ex

MYIP=$(cat /etc/hosts | grep $HOSTNAME | awk '{ print $1 }')

echo "$MYIP 9092 localhost 9092" > /etc/rinetd.conf
echo "logfile /proc/self/fd/1" >> /etc/rinetd.conf

service rinetd start

netstat -tap

# with GKE your gcloud credentials will probably not be renewable inside the container
cat ~/.kube/config | grep expiry

kubectl -n $FORWARD_NAMESPACE port-forward $FORWARD_POD $FORWARD_PORT