Skip to content

Instantly share code, notes, and snippets.

View stewartshea's full-sized avatar

Shea Stewart stewartshea

View GitHub Profile
@stewartshea
stewartshea / check_for_available_helm_chart_updates.txt
Created August 4, 2023 10:49
Check for Available Helm Chart Updates
namespace="${NAMESPACE}" context="${CONTEXT}"; helm_releases=$(kubectl get HelmRelease.helm.toolkit.fluxcd.io -n "$namespace" --context "$context" -o json | jq -r '.items[] | .metadata.name'); echo "$helm_releases" | while IFS= read -r release; do chart_details=$(kubectl get HelmRelease.helm.toolkit.fluxcd.io "$release" -n "$namespace" --context "$context" -o json | jq -r '.spec.chart.spec // empty'); if [[ -n "$chart_details" ]]; then chart_kind=$(echo "$chart_details" | jq -r '.sourceRef.kind // empty'); chart_name=$(echo "$chart_details" | jq -r '.chart // empty'); chart_source_name=$(echo "$chart_details" | jq -r '.sourceRef.name // empty'); chart_namespace=$(echo "$chart_details" | jq -r '.sourceRef.namespace // empty'); chart_version=$(echo "$chart_details" | jq -r '.version // "N/A"'); if [[ "$chart_kind" == "HelmRepository" && -n "$chart_name" && -n "$chart_namespace" ]]; then repo_url=$(kubectl get helmrepositories.source.toolkit.fluxcd.io "$chart_source_name" -n "$chart_namespace" --context "$contex
@stewartshea
stewartshea / find_ingress_owner_and_service_health.txt
Created July 15, 2023 13:22
Find Ingress Owner and Service Health
namespace="${NAMESPACE}"; context="${CLUSTER}"; ingress="${INGRESS_OBJECT_NAME}"; echo "Ingress: $ingress"; health_status="NA"; services=(); backend_services=$(kubectl get ingress "$ingress" -n "$namespace" --context "$context" -ojsonpath='{range .spec.rules[*].http.paths[*]}{.backend.service.name}{" "}{.backend.service.port.number}{"\n"}{end}'); IFS=$'\n'; for line in $backend_services; do service=$(echo "$line" | cut -d " " -f 1); port=$(echo "$line" | cut -d " " -f 2); if [ -n "$service" ] && [ -n "$port" ]; then echo "Backend Service: $service, Port: $port"; service_exists=$(kubectl get service "$service" -n "$namespace" --context "$context" -ojsonpath='{.metadata.name}'); if [ -z "$service_exists" ]; then health_status="Unhealthy"; echo "Validation: Service $service does not exist"; else endpoint_pods=$(kubectl get endpoints "$service" -n "$namespace" --context "$context" -ojsonpath='{range .subsets[*].addresses[*]}- Pod Name: {.targetRef.name}, Pod IP: {.ip}\n{end}'); if [ -z "$endpoint_pods" ]; then he
@stewartshea
stewartshea / check_missing_or_risky_poddisruptionbudget_policies.txt
Created July 13, 2023 20:53
Check Missing or Risky PodDisruptionBudget Policies
context="${CONTEXT}"; namespace="${NAMESPACE}"; check_health() { local type=$1; local name=$2; local replicas=$3; local selector=$4; local pdbs=$(kubectl --context "$context" --namespace "$namespace" get pdb -o json | jq -c --arg selector "$selector" '.items[] | select(.spec.selector.matchLabels | to_entries[] | .key + "=" + .value == $selector)'); if [[ $replicas -gt 1 && -z "$pdbs" ]]; then printf "%-30s %-30s %-10s\n" "$type/$name" "" "Missing"; else echo "$pdbs" | jq -c . | while IFS= read -r pdb; do local pdbName=$(echo "$pdb" | jq -r '.metadata.name'); local minAvailable=$(echo "$pdb" | jq -r '.spec.minAvailable // ""'); local maxUnavailable=$(echo "$pdb" | jq -r '.spec.maxUnavailable // ""'); if [[ "$minAvailable" == "100%" || "$maxUnavailable" == "0" || "$maxUnavailable" == "0%" ]]; then printf "%-30s %-30s %-10s\n" "$type/$name" "$pdbName" "Risky"; elif [[ $replicas -gt 1 && ("$minAvailable" != "100%" || "$maxUnavailable" != "0" || "$maxUnavailable" != "0%") ]]; then printf "%-30s %-30s %-10s\n" "$ty
@stewartshea
stewartshea / list_imagepullbackoff_events_and_test_path_and_tags.txt
Created July 13, 2023 02:57
List ImagePullBackOff Events and Test Path and Tags
NAMESPACE=${NAMESPACE}; POD_NAME="skopeo-pod"; CONTEXT="${CONTEXT}"; events=$(kubectl get events -n $NAMESPACE --context=$CONTEXT -o json | jq --arg timestamp "$(date -u -v -5M +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -d "-5 minutes" +"%Y-%m-%dT%H:%M:%SZ")" '.items[] | select(.lastTimestamp > $timestamp)'); if [[ ! -z "${events-unset}" ]]; then image_pull_backoff_events=$(echo "$events" | jq -s '[.[] | select(.reason == "BackOff") | .message] | .[]'); else echo "No events found in the last 5 minutes"; exit; fi; if [[ $image_pull_backoff_events =~ "Back-off pulling image" ]]; then echo "Running Skopeo Pod"; kubectl run $POD_NAME --restart=Never -n $NAMESPACE --context=$CONTEXT --image=quay.io/containers/skopeo:latest --command -- sleep infinity && echo "Waiting for the $POD_NAME to be running..." && kubectl wait --for=condition=Ready pod/$POD_NAME -n $NAMESPACE --context=$CONTEXT ; else echo "No image pull backoff events found"; exit; fi; while IFS= read -r event; do echo "Found BackOff with message: $even
@stewartshea
stewartshea / test_service_account_access_to_kubernetes_api_server.txt
Created July 11, 2023 23:05
Test Service Account Access to Kubernetes API Server
apiserver=https://kubernetes.default.svc; namespace=${NAMESPACE}; context=${CONTEXT}; resource=""; serviceaccount=default; kubectl run curl-pod --image=curlimages/curl:latest --restart=Never --overrides="{ \"spec\": { \"serviceAccountName\": \"$serviceaccount\" } }" -n $namespace --context=$context --command -- sleep infinity && echo "Waiting for the curl-pod to be running..." && kubectl wait --for=condition=Ready pod/curl-pod --timeout=20s -n $namespace --context=$context && TOKEN=$(kubectl exec curl-pod -n $namespace --context=$context -- cat /var/run/secrets/kubernetes.io/serviceaccount/token) && echo "Performing a curl request to the Kubernetes API..." && kubectl exec curl-pod -n $namespace --context=$context -- curl -s -k -H "Authorization: Bearer $TOKEN" $apiserver$resource && echo "Cleaning up..." && kubectl delete pod curl-pod -n $namespace --context=$context && echo "Done"
@stewartshea
stewartshea / check_for_rwo_persistent_volume_node_attachment_issues.txt
Created July 10, 2023 11:39
Check for RWO Persistent Volume Node Attachment Issues
NAMESPACE="${NAMESPACE}"; CONTEXT="${CONTEXT}"; PODS=$(kubectl get pods -n $NAMESPACE --context=$CONTEXT -o json); for pod in $(jq -r '.items[] | @base64' <<< "$PODS"); do _jq() { jq -r ${1} <<< "$(base64 --decode <<< ${pod})"; }; POD_NAME=$(_jq '.metadata.name'); POD_NODE_NAME=$(kubectl get pod $POD_NAME -n $NAMESPACE --context=$CONTEXT -o custom-columns=:.spec.nodeName --no-headers); PVC_NAMES=$(kubectl get pod $POD_NAME -n $NAMESPACE --context=$CONTEXT -o jsonpath='{.spec.volumes[*].persistentVolumeClaim.claimName}'); for pvc_name in $PVC_NAMES; do PVC=$(kubectl get pvc $pvc_name -n $NAMESPACE --context=$CONTEXT -o json); ACCESS_MODE=$(jq -r '.spec.accessModes[0]' <<< "$PVC"); if [[ "$ACCESS_MODE" == "ReadWriteOnce" ]]; then PV_NAME=$(jq -r '.spec.volumeName' <<< "$PVC"); STORAGE_NODE_NAME=$(jq -r --arg pv "$PV_NAME" '.items[] | select(.status.volumesAttached != null) | select(.status.volumesInUse[] | contains($pv)) | .metadata.name' <<< "$(kubectl get nodes --context=$CONTEXT -o json)"); echo "-----"; if
@stewartshea
stewartshea / list_images_and_tags_for_every_container_in_running_pods.txt
Last active July 7, 2023 19:38
List Images and Tags for Every Container in Running Pods
kubectl get pods --context=${CONTEXT} -n ${NAMESPACE} --field-selector=status.phase==Running -o=json | jq -r '.items[] | "---", "pod_name: " + .metadata.name, "Status: " + .status.phase, "containers:", (.spec.containers[] | "- container_name: " + .name, " image_path: " + (.image | split(":")[0]), " image_tag: " + (.image | split(":")[1])), "---"'
@stewartshea
stewartshea / fetch_ingress_object_health_in_namespace.txt
Created July 6, 2023 20:19
Fetch Ingress Object Health in Namespace
namespace="${NAMESPACE}"; context="${CONTEXT}"; for ingress in $(kubectl get ingress -n "$namespace" --context "$context" -ojsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do echo "Ingress: $ingress"; health_status="NA"; services=(); backend_services=$(kubectl get ingress "$ingress" -n "$namespace" --context "$context" -ojsonpath='{range .spec.rules[*].http.paths[*]}{.backend.service.name}{"|"}{.backend.service.port.number}{"\n"}{end}'); while IFS='|' read -r service port; do if [ -n "$service" ] && [ -n "$port" ]; then echo "Backend Service: $service, Port: $port"; service_exists=$(kubectl get service "$service" -n "$namespace" --context "$context" -ojsonpath='{.metadata.name}'); if [ -z "$service_exists" ]; then health_status="Unhealthy"; echo "Validation: Service $service does not exist"; else endpoint_pods=$(kubectl get endpoints "$service" -n "$namespace" --context "$context" -ojsonpath='{range .subsets[*].addresses[*]}- Pod Name: {.targetRef.name}\n Pod IP: {.ip}\n{end}'); if [ -z "$endpoint_p
@stewartshea
stewartshea / find_failed_certificate_requests_and_identify_issues.txt
Last active July 6, 2023 20:16
Find Failed Certificate Requests and Identify Issues
kubectl get certificaterequests.cert-manager.io --context=${CONTEXT} -n ${NAMESPACE} -o json | jq -r '.items[] | select(.status.conditions[] | select(.type == "Ready" and .status != "True")) | {certRequest: .metadata.name, certificate: (.metadata.ownerReferences[].name), issuer: .spec.issuerRef.name, readyStatus: (.status.conditions[] | select(.type == "Ready")).status, readyMessage: (.status.conditions[] | select(.type == "Ready")).message, approvedStatus: (.status.conditions[] | select(.type == "Approved")).status, approvedMessage: (.status.conditions[] | select(.type == "Approved")).message} | "---\nCertificateRequest: \(.certRequest)", "Certificate: \(.certificate)", "Issuer: \(.issuer)", "Ready Status: \(.readyStatus)", "Ready Message: \(.readyMessage)", "Approved Status: \(.approvedStatus)", "Approved Message: \(.approvedMessage)"'
@stewartshea
stewartshea / troubleshoot_unready_kustomizations_with_fluxcd.txt
Last active July 18, 2023 11:54
Troubleshoot Unready Kustomizations with FluxCD
kubectl get Kustomization.kustomize.toolkit.fluxcd.io -n ${NAMESPACE} --context ${CONTEXT} -o json | jq -r '.items[] | select (.status.conditions[] | select(.type == "Ready" and .status == "False")) | "---\nKustomization Name: \(.metadata.name)\n\nReady Status: \(.status.conditions[] | select(.type == "Ready") | "\n ready: \(.status)\n message: \(.message)\n reason: \(.reason)\n last_transition_time: \(.lastTransitionTime)")\n\nReconcile Status:\(.status.conditions[] | select(.type == "Reconciling") |"\n reconciling: \(.status)\n message: \(.message)")\n---\n"'