Skip to content

Instantly share code, notes, and snippets.

@semachka
Last active Nov 10, 2021
Embed
What would you like to do?
AWS
# Setup Section
cat >> ~./zsh_profile
export PATH=~/kafka-3.0.0-src/bin:$PATH
export CLASSPATH=/Users/skopachevsky/Documents/Tools/kafka/aws-msk-iam-auth-1.1.0-all.jar
export BOOTSTRAP_SERVERS=$(aws kafka get-bootstrap-brokers --cluster-arn arn:aws:kafka:us-east-1:535059991238:cluster/brandvno-dev-kafka/7cd34252-c89b-4f31-82a6-48d618368f47-2 --output text)
export ARN=$(arn:aws:kafka:us-east-1:535059991238:cluster/brandvno-dev-kafka/7cd34252-c89b-4f31-82a6-48d618368f47-2)
# Get Tipic
./bin/kafka-log-dirs.sh --bootstrap-server b-2.brandvno-dev-kafka.pwjp0a.c2.kafka.us-east-1.amazonaws.com:9094 --topic-list 'op.dev.telecom-identity-group' --describe--command-config ./bin/client-ssl.properties
# can have 1 broker, it will fetch everything from cluster including all other brokers
./bin/kafka-log-dirs.sh --bootstrap-server b-2.brandvno-dev-kafka.pwjp0a.c2.kafka.us-east-1.amazonaws.com:9094 --describe --command-config ./bin/client-ssl.properties \
| grep '^{' | jq '.brokers[].logDirs[].partitions[]' | jq 'select(.size>0)' | jq -r .partition | sed 's/-[0-9]*$//' | sort -u
############### SCRIPT BELLOW ###############
#!/bin/bash
echo "> Beginning Detox Script"
BROKERS=($(aws --region us-east-1 kafka get-bootstrap-brokers --cluster-arn "$ARN" --output text | tr "," "\n"))
BROKER=${BROKERS[$RANDOM % ${#BROKERS[@]} ]}
partitions=$( ./kafka_2.13-3.0.0/bin/kafka-log-dirs.sh \
--bootstrap-server $BROKER \
--describe --command-config ./kafka_2.13-3.0.0/bin/client-ssl.properties | grep '^{' | jq '.brokers[].logDirs[].partitions[]')
used_topics=($(echo $partitions | jq 'select(.size > 0)' | jq -r .partition | sed 's/-[0-9]*$//' | sort -u | tr " " "\n"))
topics=($(echo $partitions | jq -r .partition | sed 's/-[0-9]*$//' | sort -u | tr " " "\n"))
declare -a empty_topics
# Filter out empty topics across partitions
for topic in "${topics[@]}"; do
if ! [[ ${used_topics[*]} =~ "$topic" ]]; then
if ! [[ ${empty_topics[*]} =~ "$topic" ]]; then
empty_topics+=($topic)
fi
fi
done
echo "Empty Topics: " ${empty_topics[*]}
# Deleting Topics
for topic in "${empty_topics[@]}"; do
echo "Deleting topic: "$topic
./kafka_2.13-3.0.0/bin/kafka-topics.sh --bootstrap-server ${BROKER} --command-config ./kafka_2.13-3.0.0/bin/client-ssl.properties \
--topic $topic \
--delete
done
echo "> Finished Running Detox Script"
# ### Manually Running From Terminal ###
#
#
# ## Get List of Brokers
# export BROKERS=$(aws kafka get-bootstrap-brokers --cluster-arn ${ARN} --output text)
#
# ## Fetch topic partitions
# ./bin/kafka-log-dirs.sh --bootstrap-server b-6.brandvno-dev-kafka.pwjp0a.c2.kafka.us-east-1.amazonaws.com:9094 --describe --command-config ./bin/client-ssl.properties \
# | grep '^{' | jq '.brokers[].logDirs[].partitions[]' | grep -C3 'stephen-test-topic'
#
# ## Delete single topic
# ./bin/kafka-topics.sh --bootstrap-server b-6.brandvno-dev-kafka.pwjp0a.c2.kafka.us-east-1.amazonaws.com:9094 --command-config ./bin/client-ssl.properties \
# --topic 'stephen-test-topic' \
# --delete
# Fetch Secret Manager
- aws --region us-east-1 secretsmanager get-secret-value --secret-id { SecretName }
# Fetch Parameter Store
- aws ssm get-parameter --region us-east-1 --name "/gitlab/reflect/api_key" --query 'Parameter.Value' --output text --with-decryption
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment