Skip to content

Instantly share code, notes, and snippets.

@nshores
Last active February 2, 2022 02:07
Show Gist options
  • Save nshores/31a4f879f44c379a85a60e5ea5a8ba25 to your computer and use it in GitHub Desktop.
Save nshores/31a4f879f44c379a85a60e5ea5a8ba25 to your computer and use it in GitHub Desktop.
#!/bin/bash
# Example usage: ./find_misconfigured_autoscaler.sh -p autogrid-dev -r us-east-2
# Your kubeconfig needs to have all of the AG clusters in it, and the names context must match AWS
# set the F flag to true to fix ASG tagging (brute force method)
while getopts p:r:c:f: flag
do
case "${flag}" in
p) AWS_PROFILE=${OPTARG};;
r) REGIONS=${OPTARG};;
c) CLUSTER_TARGET=${OPTARG};;
f) FIX_TAG=${OPTARG};;
esac
done
if [ "$AWS_PROFILE" = "" ]
then
echo "Please specify AWS profile"
exit
fi
#Make sure region is set when cluster is set
if [ "$CLUSTER_TARGET" ]
then
if [ -z "$REGIONS" ]
then
echo "Please specify region"
exit
fi
fi
#Get list of AWS REGIONS
if [ "$REGIONS" = "" ]
then
echo "Getting Regions"
REGIONS=$(aws ec2 describe-regions --region us-east-2 --profile $AWS_PROFILE --output text | cut -f4 | sort)
echo -e "Region List:\n$REGIONS"
fi
#Main Loop
for region in $REGIONS; do
echo "Getting EKS Clusters in $region"
if [ "$CLUSTER_TARGET" ]
then
CLUSTERS=$CLUSTER_TARGET
else
CLUSTERS=$(aws eks list-clusters --profile $AWS_PROFILE --output text --region $region | cut -f2)
fi
echo "$CLUSTERS"
for cluster in $CLUSTERS; do
echo "--------------------------"
echo "Getting $cluster ASG List"
cluster_asg_list=$(aws autoscaling describe-auto-scaling-groups --query "AutoScalingGroups[? Tags[? (Key=='eks:cluster-name') && Value=='$cluster']]".AutoScalingGroupName --profile $AWS_PROFILE --region $region --output text)
#count the elements
cluster_asg_count=$(echo $cluster_asg_list | wc -w)
echo -e "Current ASG List: $cluster_asg_list\n$cluster has $cluster_asg_count ASG associated"
#update kubectl
#echo 'Writing temp kubeconfig to gain access to configmap'
#aws eks update-kubeconfig --region $region --name $cluster --profile $AWS_PROFILE --kubeconfig temp_kube
echo "Setting Context to $cluster"
kubectx $cluster
#get current number of clusters the autoscaler knows about
echo -e "Getting number of clusters associated with Cluster Autoscaler"
autoscaler_asg_name=$(kubectl get configmap cluster-autoscaler-status -n kube-system -o yaml | yq e '.data.status' - | grep Name | awk '{print $2}')
#count the elements
autoscaler_asg_count=$(echo $autoscaler_asg_name | wc -w)
echo -e "Autoscaler is aware of \n$autoscaler_asg_name \nTotal Number: $autoscaler_asg_count"
if [ "$autoscaler_asg_count" != "$cluster_asg_count" ]; then
echo "Count does not match! - Check the following ASG's for tagging issues:"
problem_asg_list=$(comm -23 <(echo $cluster_asg_list | tr ' ' '\n' | sort) <(echo $autoscaler_asg_name | tr ' ' '\n' | sort))
echo $problem_asg_list
echo "ASG Tags Expected by Autoscaler:"
tags=$(kubectl get deployment/cluster-autoscaler -n kube-system -o json | jq '.spec.template.spec.containers[0].command' | grep asg | sort | uniq | cut -d "=" -f3 | cut -f1 -d'"')
#split the tags by delim
tags=$(echo $tags | awk -F',' '{print $1,$2}')
echo $tags
if [[ -n "$FIX_TAG" ]]
then
echo "Fixing Tags on ASG"
for my_asg in $cluster_asg_list; do
echo "Checking $my_asg"
#Get the tags on the ASG from AWS
aws autoscaling describe-tags --filters Name=auto-scaling-group,Values=$my_asg --profile $AWS_PROFILE --region $region --output text | grep cluster-autoscaler
#echo $asg_tag
#set k8s.io/cluster-autoscaler/enabled > true
#set k8s.io/cluster-autoscaler/$cluster_name > owned
aws autoscaling create-or-update-tags --profile $AWS_PROFILE --region $region \
--tags ResourceId=$my_asg,ResourceType=auto-scaling-group,Key="k8s.io/cluster-autoscaler/enabled",Value=true,PropagateAtLaunch=true \
ResourceId=$my_asg,ResourceType=auto-scaling-group,Key="k8s.io/cluster-autoscaler/$(echo $tags | cut -d "/" -f5)",Value=owned,PropagateAtLaunch=true \
done
fi
fi
echo "--------------------------"
done
done
#!/bin/bash
# Example usage: ./find_misconfigured_autoscaler.sh -p autogrid-dev -r us-east-2
# Your kubeconfig needs to have all of the AG clusters in it, and the names context must match AWS
# set the F flag to true to fix ASG tagging (brute force method)
while getopts p:r:c:f: flag
do
case "${flag}" in
p) AWS_PROFILE=${OPTARG};;
r) REGIONS=${OPTARG};;
c) CLUSTER_TARGET=${OPTARG};;
f) FIX_TAG=${OPTARG};;
esac
done
if [ "$AWS_PROFILE" = "" ]
then
echo "Please specify AWS profile"
exit
fi
#Make sure region is set when cluster is set
if [ "$CLUSTER_TARGET" ]
then
if [ -z "$REGIONS" ]
then
echo "Please specify region"
exit
fi
fi
#Get list of AWS REGIONS
if [ "$REGIONS" = "" ]
then
echo "Getting Regions"
REGIONS=$(aws ec2 describe-regions --region us-east-2 --profile $AWS_PROFILE --output text | cut -f4 | sort)
echo -e "Region List:\n$REGIONS"
fi
#Main Loop
for region in $REGIONS; do
echo "Getting EKS Clusters in $region"
if [ "$CLUSTER_TARGET" ]
then
CLUSTERS=$CLUSTER_TARGET
else
CLUSTERS=$(aws eks list-clusters --profile $AWS_PROFILE --output text --region $region | cut -f2)
fi
echo "$CLUSTERS"
for cluster in $CLUSTERS; do
echo find asg list
echo update asg
done
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment