Makefile

BASEDIR=$(shell basename $(PWD))
PRJ_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))

SSH_HOST=h4
GROUP=ss1
OFFSET=0

KB=
ifeq '$(shell hostname -s)' 'hadoop-prod-04'
KB=@docker run -it -v "/home/driode01/kafka-area/":/stuff/ --network=host  confluentinc/cp-kafka
CMD= --command-config /stuff/client-ssl.properties.sasl_plain.prod
ENV=prod
else
ENV=local
endif

ifeq ($(ENV), local)
CMD=--command-config client-ssl.properties.sasl_plain.$(ENV)
SERVER=--bootstrap-server localhost:9092
else
SERVER=--bootstrap-server kafka-prod-01.uit.tufts.edu:9092
endif

## help: print this help message
.PHONY: help
help:
	@echo 'Usage:'
	@sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' | sed -e 's/^/ /'

## kafka/up: print this help message
.PHONY: kafka/up
kafka/up:
	@make zk/start && \
	echo "Waiting for 5 sec for zookeeper to start ..." && \
	sleep 5 && \
	make kafka/start

## kafka/down: print this help message
.PHONY: kafka/down
kafka/down:
	@make kafka/stop && \
	echo "Waiting for 10 sec for kafka to stop ..." && \
	sleep 10 && \
	make zk/stop

# ! Important
# Kafka needs to finish the shutdown process before the zookeepers shutdown.
# https://github.com/wurstmeister/kafka-docker/issues/389#issuecomment-416884988
## kafka/start: start kafka
.PHONY: kafka/start
kafka/start:
	kafka-server-start /opt/homebrew/etc/kafka/server.properties

## kafka/stop: stop kafka
.PHONY: kafka/stop
kafka/stop:
	kafka-server-stop /opt/homebrew/etc/kafka/server.properties

## zk/{start,stop}: * zookeeper
.PHONY: zk/%
zk/%:
	zkserver $*

## kafka/install: install kafka and zookeeper
.PHONY: kafka/install
kafka/install:
	brew install zookeeper kafka

## kafka/uninstall: uninstall kafka and zookeeper
.PHONY: kafka/uninstall
kafka/uninstall:
	make kafka/stop; sleep 3 ; make zk/stop
	brew uninstall zookeeper kafka
	rm -rf /opt/homebrew/etc/zookeeper
	rm -rf /opt/homebrew/var/log/zookeeper
	rm -rf /opt/homebrew/var/run/zookeeper
	rm -rf /opt/homebrew/etc/kafka
	rm -rf /opt/homebrew/var/log/kafka
	rm -rf /opt/homebrew/var/lib/kafka-logs

## kafka/pids: show zk and kafka pids
.PHONY: kafka/pids
kafka/pids:
	@ps -fe | grep kafka | grep -v grep |awk '{print $$2}'
	@ps -fe | grep zookeeper | grep -v grep | awk '{print $$2}'

## kafka/topics: show kafka topics
.PHONY: kafka/topics
kafka/topics:
	$(KB) kafka-topics $(SERVER)  --list  $(CMD)

## kafka/groups: show kafka groups
.PHONY: kafka/groups
kafka/groups:
	$(KB) kafka-consumer-groups  $(SERVER) --list $(CMD)

## kafka/create-topic-%: create topic
.PHONY: kafka/create-topic-%
kafka/create-topic-%:
	$(KB) kafka-topics \
	$(SERVER) \
	--create \
	--replication-factor 1 \
	--partitions 1 \
	--topic $* \
	$(CMD)

## kafka/last-offset-%: show last offset in topic %
.PHONY: kafka/last-offset-%
kafka/last-offset-%:
	$(KB) kafka-consumer-groups \
	$(SERVER) \
	--group $(GROUP) \
	--topic $*  \
	--reset-offsets \
	--to-latest \
	--execute \
	$(CMD)

## kafka/lag-%: show lag for group %
.PHONY: kafka/lag-%
kafka/lag-%:
	$(KB) kafka-consumer-groups $(SERVER) --describe --group $* $(CMD)

## kcat/tail-%: tail on topic
.PHONY: kcat/tail-%
kcat/tail-%:
	kcat -F ./kcat.$(ENV).config -t $*

## kcat/last-%: last topic use LAST=# and GROUP=group_name
.PHONY: kcat/last-%
kcat/last-%:
	kcat -F ./kcat.$(ENV).config -C -t $* -p 0 -o -$(LAST) -e -G $(GROUP)

## kcat/info-%: info topic
.PHONY: kcat/info-%
kcat/info-%:
	kcat -F kcat.$(ENV).config -L -t $*

## kcat/produce-%: produce on topic
.PHONY: kcat/produce-%
kcat/produce-%:
	kcat -P -F kcat.$(ENV).config -t $* -

## python
setup:  .venv deps

.venv:
	virtualenv -p python3 .venv

activate:
	@echo ". ./.venv/bin/activate"

deps:
	brew install librdkafka kcat
	pip install pylint flake8 pep8 black confluent-kafka

virtualenv:
	pip3.9 install virtualenv

## rsync: rsync source code to tufts
.PHONY: rsync
rsync:
	rsync -avz \
	--rsync-path=/usr/bin/rsync \
	--exclude=".git" \
	--exclude=".venv" \
	--delete ../$(BASEDIR) $(SSH_HOST):.