Last active
December 4, 2021 13:42
-
-
Save bh7cw/e59a508cf7ace8dd99cbd55d39889f58 to your computer and use it in GitHub Desktop.
OSD-7443 script
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# set -x | |
echo "INFO: start putting the cluster into an imbalanced state" | |
# target workloads to rebalance | |
TARGET_PODS="alertmanager-main|prometheus-k8s|prometheus-adapter|splunkforwarder-deployment" | |
# check all the target pods on each infra node | |
for NODE in $(oc get node -l node-role.kubernetes.io/infra= --no-headers | awk '{print $1}'); do | |
NUM=$(oc describe node ${NODE} | egrep ${TARGET_PODS} | wc -l) | |
echo "INFO: there are ${NUM} target pods running on infra node ${NODE}" | |
done | |
# get the first infra node | |
INFRA_NODE=$(oc get node -l node-role.kubernetes.io/infra= --no-headers | awk '{print $1}' | head -n 1) | |
# get the infra node with the maximum target pods INFRA_NODE | |
for NODE in $(oc get node -l node-role.kubernetes.io/infra= --no-headers | awk '{print $1}'); do | |
# set infra nodes NotSchedulable | |
oc adm cordon ${NODE} | |
if [ $(oc describe node ${INFRA_NODE} | egrep ${TARGET_PODS} | wc -l) -lt $(oc describe node ${NODE} | egrep ${TARGET_PODS} | wc -l) ]; then | |
INFRA_NODE=${NODE} | |
fi | |
done | |
# set INFRA_NODE Schedulable | |
oc adm uncordon ${INFRA_NODE} | |
# delete target pods on infra nodes except INFRA_NODE | |
for NODE in $(oc get node -l node-role.kubernetes.io/infra= --no-headers | awk '{print $1}'); do | |
if [ ${INFRA_NODE} != ${NODE} ]; then | |
if [ $(oc describe node ${NODE} | egrep ${TARGET_PODS} | wc -l) -ne 0 ]; then | |
# delete the target workloads on infra nodes | |
for POD in $(oc describe node ${NODE} | egrep ${TARGET_PODS} | awk '{print $2}'); do | |
NS=$(oc get pods --all-namespaces -o wide --field-selector spec.nodeName=${NODE} | grep ${POD} | awk '{print $1}') | |
VOLUME_NAME="" | |
if [[ ${POD} == "alertmanager-main-"* ]]; then | |
VOLUME_NAME="alertmanager-data" | |
fi | |
if [[ ${POD} == "prometheus-k8s-"* ]]; then | |
VOLUME_NAME="prometheus-data" | |
fi | |
if [[ ${VOLUME_NAME} != "" ]]; then | |
# delete PVC | |
PVC=$(oc get pod -n $NS $POD -o jsonpath='{.spec.volumes[?(@.name=="'$VOLUME_NAME'")].persistentVolumeClaim.claimName}') | |
echo "INFO: Deleting PVC $PVC in NS $NS" | |
oc delete pvc --wait=false -n $NS $PVC | |
fi | |
echo "INFO: Deleting POD $POD in NS $NS" | |
oc delete pod ${POD} -n ${NS} | |
done | |
fi | |
fi | |
done | |
# check all the target pods are scheduled on INFRA_NODE | |
for NODE in $(oc get node -l node-role.kubernetes.io/infra= --no-headers | awk '{print $1}'); do | |
NUM=$(oc describe node ${NODE} | egrep ${TARGET_PODS} | wc -l) | |
echo "INFO: there are ${NUM} target pods running on infra node ${NODE}" | |
done | |
# make the infra nodes schedulable | |
for NODE in $(oc get node -l node-role.kubernetes.io/infra= --no-headers | awk '{print $1}'); do | |
if [ ${INFRA_NODE} != ${NODE} ]; then | |
oc adm uncordon ${NODE} | |
fi | |
done | |
echo "INFO: the cluster is imbalanced!" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment