kubectl cheatsheet for container probes

kubectl-probes.sh

# All containers, probes, restarts, duration of startup, and an assessment of probe settings relative to the startup time:

cat<<EOF > podlist.gotemplate 
{{- range .items -}}{{\$pod := .}}
    {{- .metadata.name -}}{{- "," -}}
    {{- .metadata.namespace -}}{{- "," -}}
    {{- range .status.conditions -}}
        {{- if eq .type "PodScheduled" -}}
            {{- .lastTransitionTime -}}
        {{- end -}}
    {{- end -}}
    {{- "," -}}
    {{- range .status.conditions -}}
        {{- if eq .type "Initialized" -}}
            {{- .lastTransitionTime -}}
        {{- end -}}
    {{- end -}}
    {{- "," -}}
    {{- range .status.conditions -}}
        {{- if eq .type "ContainersReady" -}}
            {{- .lastTransitionTime -}}
        {{- end -}}
    {{- end -}}
    {{- "," -}}
    {{- range .status.conditions -}}
        {{- if eq .type "Ready" -}}
            {{- .lastTransitionTime -}}
        {{- end -}}
    {{- end -}}
    {{- "," -}}
    {{- range .spec.containers -}}{{\$container := .}}
        {{- .name -}}
        {{- ",runtime," -}}
        {{- .startupProbe.failureThreshold -}}{{- "," -}}
        {{- .startupProbe.successThreshold -}}{{- "," -}}
        {{- .startupProbe.periodSeconds -}}{{- "," -}}
        {{- .startupProbe.timeoutSeconds -}}{{- "," -}}
        {{- .startupProbe.initialDelaySeconds -}}{{- "," -}}
        {{- .readinessProbe.failureThreshold -}}{{- "," -}}
        {{- .readinessProbe.successThreshold -}}{{- "," -}}
        {{- .readinessProbe.periodSeconds -}}{{- "," -}}
        {{- .readinessProbe.timeoutSeconds -}}{{- "," -}}
        {{- .readinessProbe.initialDelaySeconds -}}{{- "," -}}
        {{- .livenessProbe.failureThreshold -}}{{- "," -}}
        {{- .livenessProbe.successThreshold -}}{{- "," -}}
        {{- .livenessProbe.periodSeconds -}}{{- "," -}}
        {{- .livenessProbe.timeoutSeconds -}}{{- "," -}}
        {{- .livenessProbe.initialDelaySeconds -}}{{- "," -}}
        {{- range \$pod.status.containerStatuses -}}
            {{- if eq .name \$container.name -}}
                {{- .started -}}{{- "," -}}
                {{- .ready -}}{{- "," -}}
                {{- .restartCount -}}{{- "," -}}
                {{- .lastState.terminated.finishedAt -}}{{- "," -}}
                {{- if .state.running.startedAt -}}
                    {{- .state.running.startedAt -}}
                {{- else -}}
                    {{- .state.terminated.startedAt -}}
                {{- end -}}
                {{- "," -}}
                {{- .state.terminated.reason -}}
            {{- end -}}
        {{- end -}}
        {{"\n"}}
    {{- end -}}
{{- end -}}
EOF

echo "pod name, pod namespace, pod scheduled, pod initialized, pod containers ready, pod ready, startup probe failure threshold, startup probe success threshold, startup probe period, startup probe failure timeout, startup probe initial delay, readiness probe failure threshold, readiness probe success threshold, readiness probe period, readiness probe failure timeout, readiness probe initial delay, liveness probe failure threshold, liveness probe success threshold, liveness probe period, liveness probe failure timeout, liveness probe initial delay, started, ready, restarts, running at, terminated at, reason, startup duration"
kubectl get pod -o go-template-file=podlist.gotemplate | \
sed "s|<no value>|nil|g" | \
while read -r line
do
    cell_count=1
    pod_initialized=0
    restart_count=0
    last_terminated=0
    readiness_threshold=1
    readiness_period=1
    readiness_initial_delay=0
    readiness_toleration=-1
    duration=0
    for cell in ${line//,/ }
    do
        if [[ ${cell} == *-*-*T*:*:* ]]; then
            timestamp=$(date -d "${cell}" "+%s")
            cell="${timestamp}"
        fi
        if [ ${cell_count} -eq 4 ]; then
            pod_initialized=${timestamp}
        fi
        if [ ${cell_count} -eq 26 ] && [ "${cell}" != "nil" ]; then
            restart_count=${cell}
        fi
        if [ ${cell_count} -eq 14 ]; then
            readiness_threshold=${cell}
        fi
        if [ ${cell_count} -eq 16 ]; then
            readiness_period=${cell}
        fi
        if [ ${cell_count} -eq 18 ]; then
            readiness_initial_delay=${cell}
        fi
        if [ ${cell_count} -eq 27 ] && [ "${cell}" != "nil" ];  then
            last_terminated=${timestamp}
        fi
        if [ ${cell_count} -eq 28 ]; then
            if [ "${restart_count}" -gt 0 ]; then 
                if [ "${last_terminated}" -gt 0 ]; then
                    duration=$((timestamp-last_terminated))
                else
                    duration="-"
                fi
            else
                duration=$((timestamp-pod_initialized))
            fi
        fi
        echo -n "${cell},"
        cell_count=$((cell_count+1))
    done

    ratio="-"
    readiness_toleration=-1
    assessment="none"
    if [ "${readiness_threshold}" != "nil" ] || [ "${readiness_period}" != "nil" ] || [ "${readiness_initial_delay}" != "nil" ]; then
        if [ "${readiness_threshold}" == "nil" ]; then
            readiness_threshold=1
        fi
        if [ "${readiness_period}" == "nil" ]; then
            readiness_period=1
        fi
        if [ "${readiness_initial_delay}" == "nil" ]; then
            readiness_period=0
        fi
        assessment="none"
        readiness_toleration=$((readiness_initial_delay+readiness_period*readiness_threshold))
        if [ "${duration}" != "-" ]; then
            ratio=$((duration*100/readiness_toleration))
            if [ ${ratio} -gt 75 ]; then
                assessment="readiness tolerance too low"
            elif [ ${ratio} -lt 25 ]; then
                assessment="readiness tolerance too high"
            fi
        fi
    fi

    echo "${readiness_toleration},${duration},${ratio},${assessment}"
done