Skip to content

Instantly share code, notes, and snippets.

@gkspranger
Created May 18, 2023 19:13
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save gkspranger/69bc9eea2b80c3485eafc116ab995eac to your computer and use it in GitHub Desktop.
Save gkspranger/69bc9eea2b80c3485eafc116ab995eac to your computer and use it in GitHub Desktop.
#!/usr/bin/env bash
set -e
sanitize_nomad () {
nomad system gc
nomad system reconcile summaries
}
deregister_instance () {
local dia=${1} # arn
local dii=${2} # instance
echo -e "deregistering instance ${dii} from ${dia}\n"
aws elbv2 deregister-targets \
--target-group-arn "${dia}" \
--targets Id="${dii}"
echo -e "\n\n"
}
wait_until_terminated () {
local wuti=${1} # instance
while true; do
sleep 10
local isn=$(aws ec2 describe-instances --instance-id "${wuti}" \
| jq -r '.Reservations[].Instances[0].State.Name')
echo -e "instance ${wuti} state name is ${isn}\n"
sanitize_nomad
if [[ "${isn}" == "terminated" ]]; then
sleep 30
break
fi
done
}
INSTANCES=($(aws ec2 describe-instances \
--filters "Name=tag:Name,Values=my-${ENVIRONMENT}-${SERVER_NAME}" \
"Name=instance-state-name,Values=running" | \
jq -r '.Reservations[].Instances[] | [.LaunchTime, .InstanceId, .PrivateDnsName] | @csv' | \
sed 's/"//g' | \
sort))
echo """
AWS EC2 Instances (CSV order is LaunchTime, InstanceId, PrivateDnsName):
$(printf '%s\n' "${INSTANCES[@]}")
"""
TARGET_GROUP_ARNS=($(aws elbv2 describe-target-groups \
--names my-${ENVIRONMENT}-nomad-server | \
jq -r '.TargetGroups[].TargetGroupArn'))
echo """
Target Group: ${TARGET_GROUP_ARNS[0]}
"""
# number of alive nomad servers
NOMAD_SERVERS=3
for i in "${INSTANCES[@]}"; do
a=(${i//,/ })
lt=${a[0]} # LaunchTime
ii=${a[1]} # InstanceId
pdn=${a[2]} # PrivateDnsName
echo """
LaunchTime: ${lt}
InstanceId: ${ii}
PrivateDnsName: ${pdn}
"""
# number of eligible and ready nomad nodes
nnc=$(nomad node status | grep " eligible " | grep -c ready | xargs)
# nomad node id
nni=$(nomad node status | grep "${pdn}" | awk '{print $1}' | xargs)
echo """
Nomad Server Count: ${NOMAD_SERVERS}
Nomad Node Count: ${nnc}
Nomad Node ID: ${nni}
"""
deregister_instance ${TARGET_GROUP_ARNS[0]} ${ii}
sleep 70
echo -e "disabling eligibility for ${nni}/${pdn}\n"
nomad node eligibility -disable "${nni}"
echo -e "\n\n"
sanitize_nomad
sleep 30
sanitize_nomad
echo -e "draining allocations for ${nni}/${pdn}\n"
nomad node drain -yes -m "repaving" -deadline 3m -enable "${nni}"
echo -e "\n\n"
sanitize_nomad
sleep 30
sanitize_nomad
echo -e "terminating ec2 ${ii}/${pdn}\n"
aws ec2 terminate-instances --instance-ids "${ii}"
echo -e "\n\n"
wait_until_terminated ${ii}
while true; do
sleep 10
sanitize_nomad
# number of eligible and ready nomad nodes
tnsc=$(nomad server members | grep -c " alive " | xargs)
nomad server members
echo -e "\n\n"
# compare previous number of alive nomad servers to current
if [[ ${NOMAD_SERVERS} -eq ${tnsc} ]]; then
sleep 30
break
fi
done
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment