Doesn't work:
# This doesn't work in v1.5.2, try again in v1.5.3. You could also try using 'timeout'.
# https://github.com/kubernetes/kubernetes/issues/40496
# https://github.com/kubernetes/kubernetes/issues/40207
#
# rc=0
# kubectl rollout status --watch "deployment/$app_name" || rc=$?
# if [[ $rc -ne 0 ]]; then
# echo "Rolling back:" >&2
# echo kubectl rollout undo "deployment/${app_name}"
# exit 1
# fi#!/bin/bash
set -o errexit
set -o nounset
set -o pipefail
set -o xtrace
registry=${KUBERNETES_REGISTRY:-docker.local}
app=${KUBERNETES_APP:-}
if [[ -z "$app" ]]; then
echo "KUBERNETES_APP should be defined (e.g. 'harry')" >&2
exit 1
fi
namespace=${KUBERNETES_NAMESPACE:-}
timeout=${KUBERNETES_CI_TIMEOUT_SECONDS:-300}
case $timeout in
''|*[!0-9]*) echo "KUBERNETES_CI_TIMEOUT_SECONDS should be a number" >&2 && exit 1 ;;
*) true ;;
esac
if [[ $timeout -lt 10 ]]; then
echo "KUBERNETES_CI_TIMEOUT_SECONDS should be at least 10" >&2 && exit 1
fi
kube_config=${KUBE_CONFIG:-config}
if [[ ! -e "$HOME/.kube/$kube_config" ]]; then
echo "kube config does not exist: $HOME/.kube/$kube_config" >&2
exit 1
fi
kubernetes_rollback_on_failure=${KUBERNETES_ROLLBACK_ON_FAILURE:-true}
if [[ $kubernetes_rollback_on_failure != true ]] && [[ $kubernetes_rollback_on_failure != false ]]; then
echo "\$KUBERNETES_ROLLBACK_ON_FAILURE should be true or false" >&2
exit 1
fi
image=${IMAGE:-$app}
image_tag=${IMAGE_TAG:-}
if [[ -z "$image_tag" ]]; then
image_tag="$(git log -1 --date=short --pretty=format:%cd)-$(git rev-parse --short HEAD)"
fi
kubernetes_push_latest=${KUBERNETES_PUSH_LATEST:-false}
if [[ $kubernetes_push_latest != true ]] && [[ $kubernetes_push_latest != false ]]; then
echo "\$KUBERNETES_PUSH_LATEST should be true or false" >&2
exit 1
fi
k8s_push() {
docker tag "$1" "$registry/$image:$image_tag"
docker push "$registry/$image:$image_tag"
if [[ $kubernetes_push_latest == true ]]; then
docker tag "$1" "$registry/$image:latest"
docker push "$registry/$image:latest"
fi
}
kubectl_helper() {
set -o errexit
set -o nounset
set -o pipefail
if [[ -z "$namespace" ]]; then
echo "KUBERNETES_NAMESPACE should be defined (e.g. 'prod' or 'staging')" >&2
exit 1
fi
kubectl --kubeconfig "$HOME/.kube/$kube_config" --namespace "${app}-${namespace}" "$@"
}
k8s_describe() {
set -o errexit
set -o nounset
set -o pipefail
printf "\nCurrent deployment (short):\n"
kubectl_helper get deployments -a -o wide
printf "\nCurrent deployment (detailed):\n"
kubectl_helper describe deployment "$app" || true
printf "\nCurrent deployment (yaml):\n"
kubectl_helper get deployment "$app" -o yaml || true
printf "\nReplica sets:\n"
kubectl_helper get replicasets -a -o wide
printf "\nPods:\n"
kubectl_helper get pods -a -o wide
}
k8s_update_tag() {
set -o errexit
set -o nounset
set -o pipefail
if [[ $# -ne 1 ]]; then
echo "Usage: k8s_update_tag <deployment-file>" >&2
exit 1
fi
file=$1
# https://github.com/kubernetes/kubernetes/issues/33664
sed -i.old -r "s|($registry/$image):@VERSION@|\\1:${image_tag}|" "$file"
if diff "$file" "$file.old" >/dev/null; then
echo "@VERSION@ not found in $file" >&2
exit 1
fi
rm "$file.old"
sed -i -r "/^\\s*#/d" "$file"
}
k8s_apply() {
set -o errexit
set -o nounset
set -o pipefail
# 1. This does not wait for the deployment to actually finish:
# https://github.com/kubernetes/kubernetes/issues/1899
# 2. If you delete some properties you might have to use `replace` here
# 3. You should not use `replace` here:
# https://github.com/kubernetes/kubernetes/issues/11237
# https://github.com/kubernetes/kubernetes/issues/35858
kubectl_helper apply -f "$1"
}
k8s_watch_deploy() {
set -o errexit
set -o nounset
set -o pipefail
# kubectl_helper get deployment -o json | jq -r '[.spec.template.spec.containers[0].image] | join(",")'
set +o xtrace
final_exit_code=0
start_time=$(date +%s)
while true; do
deployment_json=$(kubectl_helper get "deployment/${app}" -o json || (echo "get deployment failed!" && exit 1))
replicas_spec=$(echo "$deployment_json" | jq -r ".spec.replicas" | grep -P '\d+' || (echo "Can't get .spec.replicas!" && exit 1))
replicas_status_present=$(echo "$deployment_json" | jq -r ".status.replicas" | grep -P '\d+' || (echo "Can't get .status.replicas!" && exit 1))
replicas_status_updated=$(echo "$deployment_json" | jq -r ".status.updatedReplicas" | grep -P '\d+' || echo 0)
replicas_status_unavail=$(echo "$deployment_json" | jq -r ".status.unavailableReplicas" | grep -P '\d+' || echo 0)
replicas_status_avail=$(echo "$deployment_json" | jq -r ".status.availableReplicas" | grep -P '\d+' || echo 0)
progressing=$(echo "$deployment_json" | jq -r '.status.conditions[] | select(.type == "Progressing").status' || echo "unknown")
printf "%s desired: %2d; present: %2d; available: %2d; updated: %2d; unavailable: %2d\n" \
"$(date +%T)" \
"$replicas_spec" \
"$replicas_status_present" \
"$replicas_status_avail" \
"$replicas_status_updated" \
"$replicas_status_unavail"
if [[ $progressing == "False" ]] || [[ $(($(date +%s) - start_time)) -gt "$timeout" ]]; then
if [[ $progressing == "False" ]]; then
echo "We have stopped progressing!" >&2
echo "$deployment_json" | jq '.status'
else
echo "Timed out after $timeout seconds!" >&2
fi
echo
kubectl_helper describe deployment "${app}"
if [[ $kubernetes_rollback_on_failure == true ]]; then
echo
echo "Rolling back:" >&2
kubectl_helper rollout undo "deployment/${app}"
fi
final_exit_code=1
break
fi
if [[ "$replicas_status_updated" -lt "$replicas_spec" ]] || \
[[ "$replicas_status_avail" -lt "$replicas_spec" ]] || \
[[ "$replicas_status_present" -lt "$replicas_spec" ]] || \
[[ "$replicas_status_unavail" -gt 0 ]]; then
sleep 1
continue
else
echo "Deployment successful!"
echo "New tag: $image_tag"
break
fi
done
k8s_describe
exit "$final_exit_code"
}