Skip to content

Instantly share code, notes, and snippets.

@shukla2112
Last active March 31, 2020 03:26
Show Gist options
  • Save shukla2112/035d4e633dded6441f5a9ba0743b4f08 to your computer and use it in GitHub Desktop.
Save shukla2112/035d4e633dded6441f5a9ba0743b4f08 to your computer and use it in GitHub Desktop.
fixes hpa issue while deployment is triggered
#!/bin/bash
set -e
# STEPS
# 1. Set the HPA max to current replicas
# 2. Set image
# 2.1 wait for deployment to be completed or 15 mins
# 3. Set max hpa to original number
export KUBECONFIG=/var/lib/jenkins/.kube/config ## Export your file
## variables | Change these variables as per your requirements
service_name="my-service"
image_with_tag="image_registry/image_repo:$BUILD_NUMBER"
wait_for=15 #in mins
let iterations="(${wait_for}*60)/10"
## 1. Set the HPA max to current replicas
# Get the hpa config
random=$(LC_CTYPE=C tr -d -c '[:alnum:]' </dev/urandom | head -c 8)
filename="hpa-config-$random.yaml"
rm -f $filename
kubectl get hpa $service_name -o yaml > $filename
parse_yaml() {
local prefix=$2
local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034')
sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \
-e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $1 |
awk -F$fs '{
indent = length($1)/2;
vname[indent] = $2;
for (i in vname) {if (i > indent) {delete vname[i]}}
if (length($3) > 0) {
vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
printf("%s%s%s=\"%s\"\n", "'$prefix'",vn, $2, $3);
}
}'
}
# read yaml file
eval $(parse_yaml $filename "config_")
# access yaml content
echo $config_status_currentReplicas
currentReplicas="$config_status_currentReplicas"
newMaxValue=" maxReplicas: $currentReplicas"
echo "#$newMaxValue"
oldMaxValue="$(grep " maxReplicas:" $filename)"
echo "#$oldMaxValue"
perl -i -ne "print unless /resourceVersion:/" $filename
echo $?
sleep 1
perl -pi -e "s/$oldMaxValue/$newMaxValue/g" $filename
echo $?
grep "maxReplicas" $filename
# apply hpa config
kubectl apply -f $filename
## 2. Set image
kubectl set image deployment/$service_name $service_name=$image_with_tag
## 2.1 Sleep till deployment is finished
expected_output="deployment \"$service_name\" successfully rolled out"
count=0
while :
do
((count=count+1))
echo "$count : waiting for deployment to be completed..."
sleep 10
output=$(kubectl rollout status deploy/${service_name} --watch=false)
if [[ "$output" == "$expected_output" ]]
then
echo "[Success] ::: YO ::: deployment completed ::: YO :::"
echo "resetting hpa value"
break
elif [[ $count -eq $iterations ]]
then
echo "enough time($wait_for) waited for deployment to be completed"
echo "$count : $output"
echo "resetting hpa value"
break
else
echo "[INFO] : $output"
fi
done
## 3. Set max hpa to original number
perl -i -ne "print unless /resourceVersion:/" $filename
echo $?
sleep 1
perl -pi -e "s/$newMaxValue/$oldMaxValue/g" $filename
echo $?
grep "maxReplicas" $filename
# apply hpa config
kubectl apply -f $filename
echo "SUCCESS"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment