Skip to content

Instantly share code, notes, and snippets.

@jim-cooley

jim-cooley/repartition-topic-store

Created April 29, 2022 14:54
Show Gist options
  • Save jim-cooley/0f7b590c007306930aebe944ce91c13d to your computer and use it in GitHub Desktop.
Save jim-cooley/0f7b590c007306930aebe944ce91c13d to your computer and use it in GitHub Desktop.
Download ZIP
script to repartition topic-store and other internal topics
Raw
repartition-topic-store
---
apiVersion: batch/v1
kind: Job
metadata:
annotations:
strategy.spinnaker.io/recreate: "true"
labels:
app: repartition-kafka-topic-store
team: dbre
name: repartition-kafka-topic-store
namespace: kafka
spec:
template:
metadata:
annotations:
strategy.spinnaker.io/recreate: "true"
labels:
app: repartition-kafka-topic-store
team: dbre
spec:
automountServiceAccountToken: false
dnsPolicy: ClusterFirst
containers:
- image: quay.io/strimzi/kafka:0.27.1-kafka-3.0.0
name: repartition-kafka-topic-store
args:
- >
for repartition_script in $(ls -1A /tmp/config/*.json); do \
bin/kafka-reassign-partitions.sh --bootstrap-server kafka-cluster-kafka-0.kafka-cluster-kafka-brokers.kafka.svc.cluster.local:9092 \
--reassignment-json-file $repartition_script \
--throttle 10000000 \
--execute;
sleep 10s;
bin/kafka-reassign-partitions.sh --bootstrap-server kafka-cluster-kafka-0.kafka-cluster-kafka-brokers.kafka.svc.cluster.local:9092 \
--reassignment-json-file $repartition_script \
--verify; \
done; sleep 5m;
command: [ "/bin/sh", "-c", "-x" ]
securityContext:
allowPrivilegeEscalation: false
privileged: false
readOnlyRootFilesystem: false
volumeMounts:
- name: config
mountPath: "/tmp/config"
readOnly: true
restartPolicy: Never
volumes:
- name: config
configMap:
name: repartition-kafka-config
---
apiVersion: v1
kind: ConfigMap
metadata:
annotations:
strategy.spinnaker.io/versioned: "false"
strategy.spinnaker.io/recreate: "true"
name: repartition-kafka-config
namespace: kafka
data:
topic-store.json: |
{
"version":1,
"partitions":[
{"topic":"__strimzi_store_topic", "partition":0, "replicas": [0,1,2], "log_dirs": ["any","any","any"]},
{"topic":"__strimzi-topic-operator-kstreams-topic-store-changelog", "partition":0, "replicas": [1,0,2], "log_dirs": ["any","any","any"]},
{"topic":"__consumer_offsets","partition":0,"replicas":[0,2,1],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":1,"replicas":[1,0,2],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":2,"replicas":[2,1,0],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":3,"replicas":[0,1,2],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":4,"replicas":[1,2,0],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":5,"replicas":[2,0,1],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":6,"replicas":[0,2,1],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":7,"replicas":[1,0,2],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":8,"replicas":[2,1,0],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":9,"replicas":[0,1,2],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":10,"replicas":[1,2,0],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":11,"replicas":[2,0,1],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":12,"replicas":[0,2,1],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":13,"replicas":[1,0,2],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":14,"replicas":[2,1,0],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":15,"replicas":[0,1,2],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":16,"replicas":[1,2,0],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":17,"replicas":[2,0,1],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":18,"replicas":[0,2,1],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":19,"replicas":[1,0,2],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":20,"replicas":[2,1,0],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":21,"replicas":[0,1,2],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":22,"replicas":[1,2,0],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":23,"replicas":[2,0,1],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":24,"replicas":[0,2,1],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":25,"replicas":[1,0,2],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":26,"replicas":[2,1,0],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":27,"replicas":[0,1,2],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":28,"replicas":[1,2,0],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":29,"replicas":[2,0,1],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":30,"replicas":[0,2,1],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":31,"replicas":[1,0,2],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":32,"replicas":[2,1,0],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":33,"replicas":[0,1,2],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":34,"replicas":[1,2,0],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":35,"replicas":[2,0,1],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":36,"replicas":[0,2,1],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":37,"replicas":[1,0,2],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":38,"replicas":[2,1,0],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":39,"replicas":[0,1,2],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":40,"replicas":[1,2,0],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":41,"replicas":[2,0,1],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":42,"replicas":[0,2,1],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":43,"replicas":[1,0,2],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":44,"replicas":[2,1,0],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":45,"replicas":[0,1,2],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":46,"replicas":[1,2,0],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":47,"replicas":[2,0,1],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":48,"replicas":[0,2,1],"log_dirs":["any","any","any"]},
{"topic":"__consumer_offsets","partition":49,"replicas":[1,0,2],"log_dirs":["any","any","any"]}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment