Skip to content

Instantly share code, notes, and snippets.

@ahawkins

ahawkins/await-release

Last active February 8, 2018 03:06
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save ahawkins/6457123c447030f466f10d62ed04fc83 to your computer and use it in GitHub Desktop.
Save ahawkins/6457123c447030f466f10d62ed04fc83 to your computer and use it in GitHub Desktop.
Download ZIP
Run 7 log analysis and script
Raw
await-release
#!/usr/bin/env bash
# This script is a workaround for https://github.com/kubernetes/helm/issues/2288.
# helm install --wait should do everything this script does. It should be deleted
# when the bug is fixed.
set -euo pipefail
main() {
local counter=0 release timeout pods
scratch="$(mktemp)"
while getopts ':r:t:' opt; do
case "${opt}" in
r)
release="${OPTARG}"
;;
t)
timeout="${OPTARG}"
;;
\?)
echo "Unknown argument: -${OPTARG}" 1>&2
return 1
;;
:)
echo "Option -${OPTARG} requires an argument" 1>&2
return 1
;;
esac
done
shift $((OPTIND-1))
while [ "${counter}" -lt "${timeout}" ]; do
helm ls -q --kube-context "${KUBE_CONTEXT}" > "${scratch}"
if ! grep -qF "${release}" "${scratch}" ; then
echo "${release} not found. ${counter}/${timeout} checks completed; retrying."
cat "${scratch}" 2>&1
echo 1>&2
# NOTE: The pre-increment usage. This makes the arithmatic expression
# always exit 0. The post-increment form exits non-zero when counter
# is zero. More information here: http://wiki.bash-hackers.org/syntax/arith_expr#arithmetic_expressions_and_return_codes
((++counter))
sleep 10
else
break
fi
done
if [ "${counter}" -eq "${timeout}" ]; then
echo "${release} failed to appear."
return 1
fi
counter=0
while [ "${counter}" -lt "${timeout}" ]; do
pods="$(kubectl get pods \
-l "release=${release}" \
-o 'custom-columns=NAME:.metadata.name,STATUS:.status.phase,Node:.spec.nodeName' \
-n "${KUBE_NAMESPACE}" \
--context "${KUBE_CONTEXT}" \
| tail -n +2
)"
if [ -n "${pods}" ] && echo "${pods}" | grep -qvF 'Running'; then
echo "${release} pods not ready. ${counter}/${timeout} checks completed; retrying."
echo "${pods}" | grep -vF 'Running' 1>&2
echo 1>&2
# NOTE: The pre-increment usage. This makes the arithmatic expression
# always exit 0. The post-increment form exits non-zero when counter
# is zero. More information here: http://wiki.bash-hackers.org/syntax/arith_expr#arithmetic_expressions_and_return_codes
((++counter))
sleep 10
else
echo "All ${release} pods running. Done!"
echo 1>&2
echo "${pods}" 1>&2
return 0
fi
done
echo "Release ${release} did not complete in time" 1>&2
return 1
}
main "$@"
Raw
snippet.log
++ tail -n +2
++ kubectl get pods -l release=test-efritin -o custom-columns=NAME:.metadata.name,STATUS:.status.phase,Node:.spec.nodeName -n development --context np.k8s.saltside.io
+ pods='test-efritin-kviberg-efr-admin-service-db Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-admin-service-thrift-919802698-zj1wd Pending ip-172-20-33-16.ap-south-1.compute.internal # NOTE: Pending pod
test-efritin-kviberg-efr-auth-service-db Running ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-auth-service-thrift-1214294993-2lw00 Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-chat-service-thrift-1635443225-pkjb7 Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-core-service-db Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-core-service-thrift-434229318-kp3lp Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-email-service-db Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-email-service-thrift-2610895764-c3hwn Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-image-service-db Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-image-service-thrift-3294622884-xgsml Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-inventory-service-db Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-inventory-service-thrift-20910478qm890 Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-notification-service-thrift-20869lp0k4 Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-offer-service-db Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-offer-service-thrift-3693698501-s5pl5 Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-payment-service-db Running ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-payment-service-thrift-2286136830q714d Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-review-service-db Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-review-service-thrift-2603159930-rsfg9 Pending ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-search-service-db Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-search-service-thrift-381569775-f42bq Pending ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-sms-service-db Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-sms-service-thrift-4201968210-106rk Running ip-172-20-33-16.ap-south-1.compute.internal'
+ '[' -n 'test-efritin-kviberg-efr-admin-service-db Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-admin-service-thrift-919802698-zj1wd Pending ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-auth-service-db Running ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-auth-service-thrift-1214294993-2lw00 Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-chat-service-thrift-1635443225-pkjb7 Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-core-service-db Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-core-service-thrift-434229318-kp3lp Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-email-service-db Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-email-service-thrift-2610895764-c3hwn Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-image-service-db Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-image-service-thrift-3294622884-xgsml Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-inventory-service-db Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-inventory-service-thrift-20910478qm890 Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-notification-service-thrift-20869lp0k4 Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-offer-service-db Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-offer-service-thrift-3693698501-s5pl5 Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-payment-service-db Running ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-payment-service-thrift-2286136830q714d Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-review-service-db Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-review-service-thrift-2603159930-rsfg9 Pending ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-search-service-db Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-search-service-thrift-381569775-f42bq Pending ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-sms-service-db Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-sms-service-thrift-4201968210-106rk Running ip-172-20-33-16.ap-south-1.compute.internal' ']'
+ grep -qvF Running
+ echo 'test-efritin-kviberg-efr-admin-service-db Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-admin-service-thrift-919802698-zj1wd Pending ip-172-20-33-16.ap-south-1.compute.internal # NOTE: grep -qvF against stdin with 'Pending'
test-efritin-kviberg-efr-auth-service-db Running ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-auth-service-thrift-1214294993-2lw00 Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-chat-service-thrift-1635443225-pkjb7 Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-core-service-db Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-core-service-thrift-434229318-kp3lp Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-email-service-db Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-email-service-thrift-2610895764-c3hwn Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-image-service-db Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-image-service-thrift-3294622884-xgsml Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-inventory-service-db Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-inventory-service-thrift-20910478qm890 Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-notification-service-thrift-20869lp0k4 Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-offer-service-db Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-offer-service-thrift-3693698501-s5pl5 Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-payment-service-db Running ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-payment-service-thrift-2286136830q714d Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-review-service-db Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-review-service-thrift-2603159930-rsfg9 Pending ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-search-service-db Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-image-service-db Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-image-service-thrift-3294622884-xgsml Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-inventory-service-db Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-inventory-service-thrift-20910478qm890 Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-notification-service-thrift-20869lp0k4 Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-offer-service-db Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-offer-service-thrift-3693698501-s5pl5 Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-payment-service-db Running ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-payment-service-thrift-2286136830q714d Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-review-service-db Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-review-service-thrift-2603159930-rsfg9 Pending ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-search-service-db Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-search-service-thrift-381569775-f42bq Pending ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-sms-service-db Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-sms-service-thrift-4201968210-106rk Running ip-172-20-33-16.ap-south-1.compute.internal' ']'
+ grep -qvF Running
+ echo 'test-efritin-kviberg-efr-admin-service-db Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-admin-service-thrift-919802698-zj1wd Pending ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-auth-service-db Running ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-auth-service-thrift-1214294993-2lw00 Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-chat-service-thrift-1635443225-pkjb7 Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-core-service-db Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-core-service-thrift-434229318-kp3lp Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-email-service-db Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-email-service-thrift-2610895764-c3hwn Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-image-service-db Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-image-service-thrift-3294622884-xgsml Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-inventory-service-db Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-inventory-service-thrift-20910478qm890 Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-notification-service-thrift-20869lp0k4 Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-offer-service-db Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-offer-service-thrift-3693698501-s5pl5 Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-payment-service-db Running ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-payment-service-thrift-2286136830q714d Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-review-service-db Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-review-service-thrift-2603159930-rsfg9 Pending ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-search-service-db Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-search-service-thrift-381569775-f42bq Pending ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-sms-service-db Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-sms-service-thrift-4201968210-106rk Running ip-172-20-33-16.ap-south-1.compute.internal'
+ echo 'All test-efritin pods running. Done!'
All test-efritin pods running. Done!
+ echo
+ echo 'test-efritin-kviberg-efr-admin-service-db Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-admin-service-thrift-919802698-zj1wd Pending ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-auth-service-db Running ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-auth-service-thrift-1214294993-2lw00 Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-chat-service-thrift-1635443225-pkjb7 Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-core-service-db Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-core-service-thrift-434229318-kp3lp Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-email-service-db Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-email-service-thrift-2610895764-c3hwn Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-image-service-db Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-image-service-thrift-3294622884-xgsml Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-inventory-service-db Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-inventory-service-thrift-20910478qm890 Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-notification-service-thrift-20869lp0k4 Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-offer-service-db Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-offer-service-thrift-3693698501-s5pl5 Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-payment-service-db Running ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-payment-service-thrift-2286136830q714d Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-review-service-db Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-review-service-thrift-2603159930-rsfg9 Pending ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-search-service-db Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-search-service-thrift-381569775-f42bq Pending ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-sms-service-db Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-sms-service-thrift-4201968210-106rk Running ip-172-20-33-16.ap-south-1.compute.internal' # NOTE: This is where echo ends!
test-efritin-kviberg-efr-admin-service-db Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-admin-service-thrift-919802698-zj1wd Pending ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-auth-service-db Running ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-auth-service-thrift-1214294993-2lw00 Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-chat-service-thrift-1635443225-pkjb7 Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-core-service-db Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-core-service-thrift-434229318-kp3lp Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-email-service-db Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-email-service-thrift-2610895764-c3hwn Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-image-service-db Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-image-service-thrift-3294622884-xgsml Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-inventory-service-db Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-inventory-service-thrift-20910478qm890 Running ip-172-20-50-36.ap-south-1.compute.internal
test-efritin-kviberg-efr-notification-service-thrift-20869lp0k4 Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-offer-service-db Running ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-offer-service-thrift-3693698501-s5pl5 Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-payment-service-db Running ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-payment-service-thrift-2286136830q714d Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-review-service-db Running ip-172-20-34-173.ap-south-1.compute.internal
test-efritin-kviberg-efr-review-service-thrift-2603159930-rsfg9 Pending ip-172-20-37-14.ap-south-1.compute.internal
test-efritin-kviberg-efr-search-service-db Running ip-172-20-33-16.ap-south-1.compute.internal
test-efritin-kviberg-efr-search-service-thrift-381569775-f42bq Pending ip-172-20-42-7.ap-south-1.compute.internal
test-efritin-kviberg-efr-sms-service-db Running ip-172-20-63-172.ap-south-1.compute.internal
test-efritin-kviberg-efr-sms-service-thrift-4201968210-106rk Running ip-172-20-33-16.ap-south-1.compute.internal
+ return 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment