|
curl -s https://api.github.com/repos/kubevirt/kubevirt/releases | jq '.[].tag_name | select(.|test("rc|alpha|beta")|not)' -r | sort -r | head -1 |
|
|
|
# common =============================================================================================================== |
|
|
|
## delete all pods in crash loop backoff |
|
|
|
kubectl get pods --all-namespaces --field-selector=status.phase=Running -o json | jq -r '.items[] | select(.status.containerStatuses[].state.waiting.reason=="CrashLoopBackOff") | "kubectl delete pod \(.metadata.name) -n \(.metadata.namespace)"' | sh |
|
|
|
## get k8s resources without system fields (https://stackoverflow.com/questions/43941772/get-yaml-for-deployed-kubernetes-services) |
|
kubectl -n d8-system get secret deckhouse-registry -o yaml | yq 'del(.metadata.resourceVersion, .metadata.uid, .metadata.annotations, .metadata.creationTimestamp, .metadata.selfLink, .metadata.managedFields)' |
|
|
|
## watch for deployment statuses |
|
kubectl -n echo-00-cilium get deployment -w -o json | jq '.status|del(.conditions)' |
|
|
|
## get deployment container "deckhouse" image |
|
kubectl -n d8-system get deploy/deckhouse -o json | jq '.spec.template.spec.containers[] | select( .image | test("deckhouse")) | .image' |
|
|
|
## set image for deployment |
|
kubectl -n d8-system set image deploy/deckhouse deckhouse=${IMAGE} |
|
|
|
## patch deployment |
|
kubectl patch deployment your_deployment --type merge -p '{"spec":{"template":{"spec":{"terminationGracePeriodSeconds":31}}}}' |
|
|
|
## get ds images |
|
kubectl -n d8-ingress-nginx get ds -o json | jq '.items[].spec.template.spec.containers[] | {name,image}' -c |
|
|
|
## get pods images |
|
kubectl -n d8-ingress-nginx get pods -o json | jq '.items[].spec.containers[]|{ name, image}' -c |
|
|
|
## get image for containers with name |
|
kubectl --context=dev0 -n d8-system get deploy deckhouse -o json | jq '.spec.template.spec.containers[] | select(.name="deckhosue")|.image' |
|
|
|
## get image and imageID |
|
kubectl --context=dev1 -n d8-system get pods -l app=deckhouse -o json | jq '.items[].status.containerStatuses[] | select(.name="deckhouse") | { image, imageID }' |
|
|
|
## get all pods in not running phase for system components |
|
kubectl get pods -A -o json | jq '.items[]|select(.metadata.namespace|test("(d8.+|kube-system)")) | select(.status.phase|test("Running")|not) | { ns: .metadata.namespace, name: .metadata.name, node: .spec.nodeName, phase: .status.phase }' -c |
|
|
|
## get init or containers in not ready phase |
|
kubectl get -n kube-system pods -o json | jq '.items[] | . as $item | .status | { name: $item | .metadata.name , init: [.initContainerStatuses // [] | .[] | select(.ready | not) | .name ], cont: [.containerStatuses // [] | .[] | select(.ready | not) | .name ] }' | jq '. | select((.init| length > 0) or (.cont| length >0))' |
|
|
|
## get pod info in one line and sort by ip |
|
kubectl get pods -A -o json | jq '.items[] | [.status.podIP, .spec.nodeName, .metadata.namespace, .metadata.name] | join(" ")' -r | sort -n -t . -k2,2 -k3,3 -k4,4 -k5,5 | column -t |
|
|
|
## get svc info in one line and sort by ip |
|
kubectl get svc -A -o json | jq '.items[] | [.spec.clusterIP, .status.loadBalancer.ingress[0].ip // false, .metadata.name, .metadata.namespace ] | join(" ")' -r | sort -n -t . -k2,2 -k3,3 -k4,4 -k5,5 | column -t |
|
|
|
## get headless svc |
|
kubectl get svc -A -o json | jq '.items[] | select(.spec.clusterIP=="None") | .metadata | {namespace,name}' -c |
|
|
|
## limits requests |
|
kubectl get pods -o=custom-columns=NAME:spec.containers[*].name,MEMREQ:spec.containers[*].resources.requests.memory,MEMLIM:spec.containers[*].resources.limits.memory,CPUREQ:spec.containers[*].resources.requests.cpu,CPULIM:spec.containers[*].resources.limits.cpu |
|
|
|
## exec to some shell |
|
kubectl exec -i -t -n default pt-test-pod -c test-pod "--" sh -c "clear; (bash || ash || sh)" |
|
|
|
## remove all pod in deployment in loop |
|
while true; do kubectl -n d8-ingress-nginx get pods -l name=main,app=controller -o json | jq .items[].metadata.name -r | xargs -I {} kubectl -n d8-ingress-nginx delete pod "{}"; sleep 1; done |
|
|
|
## get all pods created 10 min ago |
|
kubectl get pods --sort-by='.metadata.creationTimestamp' -A -o json | jq '.items[] | select((.status.phase == "Running") and (.status.startTime | fromdateiso8601 > now-60*10)) | {name: .metadata.name, ns: .metadata.namespace, start: .status.startTime}' -c |
|
|
|
## remove all failed pods (Complted|Error|...) |
|
kubectl delete pod -A --field-selector=status.phase==Failed |
|
kubectl get pods --field-selector 'status.phase=Failed' --all-namespaces | awk '{if ($4 != "Running") system ("kubectl -n " $1 " delete pods " $2 )}' |
|
|
|
# remove pods (spot node problem) |
|
# https://github.com/tyriis/i-see-dead-pods/tree/main |
|
kubectl get pods \ |
|
--all-namespaces \ |
|
-o go-template \ |
|
--template='{{range .items}}{{printf "%s %s %s\n" .metadata.namespace .metadata.name .status.message}}{{end}}' \ |
|
| grep "Pod was terminated in response to imminent node shutdown." \ |
|
| awk '{print $1, $2}' \ |
|
| xargs -n2 kubectl delete pod -n || true |
|
|
|
# evict just one pod |
|
kubectl drain $(kubectl -n console-review get pods -l vm.kubevirt.io/name=linux-vm-001 -o json | jq .items[0].spec.nodeName -r) --pod-selector=vm.kubevirt.io/name=linux-vm-001 --delete-emptydir-data |
|
|
|
# nodes ================================================================================================================ |
|
|
|
## get masters annotations |
|
kubectl get nodes -o json | jq '[.items[] | select(.metadata.labels."node-role.kubernetes.io/master"!=null)] | .[].metadata.annotations' |
|
|
|
## get master nodes external ip addresses |
|
kubectl get nodes -o json | jq '.items[] | select(.metadata.labels."node-role.kubernetes.io/master"!=null) | .status.addresses | .[] | select (.type=="ExternalIP") | .address' -r |
|
|
|
## get worker nodes external ip addresses |
|
kubectl get nodes -o json | jq '.items[] | select(.metadata.labels."node-role.kubernetes.io/master"==null) | .status.addresses | .[] | select (.type=="ExternalIP") | .address' -r |
|
|
|
## get nodes info |
|
kubectl get nodes -o json | jq -rc '.items[].status.addresses | reduce .[] as $node ({}; . + { "\($node.type)": "\($node.address)" })' |
|
|
|
## get pods by nodes |
|
kubectl get pods --all-namespaces -o wide --field-selector spec.nodeName=<node> |
|
|
|
## get pod name by node |
|
kubectl -n d8-cloud-instance-manager get pods -l app=fencing-agent --field-selector spec.nodeName=virtlab-pt-1 -o jsonpath="{.items[0].metadata.name}" |
|
|
|
## sort pods by node |
|
kubectl get pods -o wide --sort-by="{.spec.nodeName}" |
|
|
|
## get node topoloy lables |
|
kubectl --context dev1 get nodes -o json | jq '.items[].metadata.labels | with_entries( select(.key|contains("topology")))' |
|
|
|
# logs ================================================================================================================= |
|
|
|
## get dh logs without unneded fields |
|
kubectl -n d8-system logs --tail=100 -f deployments/deckhouse | jq -c '. | del(."task.id",."event.id")' |
|
|
|
## get dh logs and exclude info level |
|
kubectl -n d8-system logs --tail=100 -f deployments/deckhouse | jq -c '. | del(."task.id",."event.id") | select(.level|test("info")|not)' |
|
|
|
# pvcs |
|
|
|
## Remove finalizers from pvc |
|
kubectl get pv -o json | jq '.items[] | select(.status.phase=="Released") | .metadata.name' -r | xargs -I {} kubectl patch pv {} --type=json -p '[{ "op": "remove", "path": "/metadata/finalizers" }]' |
|
# events |
|
|
|
kubectl get events --field-selector involvedObject.kind=Pod |
|
|
|
kubectl get events --field-selector type!=Normal |
|
|
|
## cilium | hubble ====================================================================================================== |
|
|
|
# remove all cilium crds |
|
kubectl get crd -o json | jq '.items[].metadata | select(.name | test("cilium.io")) | .name' -r | xargs -I {} kubectl delete crd "{}" |
|
|
|
## enable hubble port-forward |
|
kubectl port-forward -n d8-cni-cilium svc/hubble-relay 4245:443 & |
|
|
|
## hubble observe |
|
hubble observe --tls --tls-allow-insecure -f --verdict DROPPED -o jsonpb | jq |
|
|
|
# etcd ================================================================================================================= |
|
|
|
## etcd get members |
|
kubectl -n kube-system exec -ti $(kubectl -n kube-system get pod -l component=etcd,tier=control-plane -o name | head -n1) -- sh -c \ |
|
"ETCDCTL_API=3 etcdctl --cacert /etc/kubernetes/pki/etcd/ca.crt --cert /etc/kubernetes/pki/etcd/ca.crt --key /etc/kubernetes/pki/etcd/ca.key --endpoints https://127.0.0.1:2379/ member list -w table" |
|
|
|
## with all members |
|
kubectl -n kube-system exec -ti $(kubectl -n kube-system get pod -l component=etcd,tier=control-plane -o name | head -n1) -- sh -c "ETCDCTL_API=3 etcdctl --cacert /etc/kubernetes/pki/etcd/ca.crt --cert /etc/kubernetes/pki/etcd/ca.crt --key /etc/kubernetes/pki/etcd/ca.key --endpoints $(kubectl -n kube-system get pod -l component=etcd,tier=control-plane -o json | jq '[.items[].status | "https://" + .podIP + ":2379" ] | join(",")' -r) member list -w table" |
|
|
|
## etcd perfomance |
|
etcdctl ... check perf |
|
|
|
## etcd check ep (get ep from member list) |
|
etcdctl ... endpoint status -w table |
|
|
|
## etcd get health |
|
etcdctl ... endpoint --cluster health |
|
|
|
# istio ================================================================================================================ |
|
|
|
## get all istio driven pods |
|
kubectl get pods -A -o json | jq '.items[].metadata | select(.annotations."sidecar.istio.io/status") | { namespace, name }' -c |
|
|
|
kubectl get pods -A -o json | jq '.items[].metadata | select(.annotations."sidecar.istio.io/status") | { namespace, name, rev: .annotations."sidecar.istio.io/status" }' -c |
|
|
|
## get all istio pods and show name namespace and istio revision |
|
kubectl get pods -A -o json | jq -r '.items[].metadata | select(.annotations."sidecar.istio.io/status") | {name, namespace, rev: .annotations."sidecar.istio.io/status" | fromjson | .revision }' |
|
|
|
## count istio resources |
|
for res in $(kubectl api-resources --namespaced --output name | grep istio); do echo "${res}"; kubectl get "${res}" -A 2>/dev/null | wc -l; done |
|
|
|
## get istio config |
|
istioctl pc all -n prober deploy/prober --context dev0 -o yaml |
|
|
|
# deckhouse ============================================================================================================ |
|
|
|
## queue len |
|
kubectl -n d8-system exec -ti deploy/deckhouse -- deckhouse-controller queue list | head -n 50 |
|
|
|
# cluster config |
|
kubectl -n d8-system exec -it deploy/deckhouse -- deckhouse-controller edit provider-cluster-configuration |
|
|
|
# converge in cluster |
|
kubectl -n d8-system exec -it deploy/terraform-auto-converger -c converger -- sh |
|
dhctl terraform check |
|
|
|
# ingress ============================================================================================================== |
|
|
|
# ingress |
|
for i in $(kubectl get -A ing -o json | jq '.items[].spec.rules[].host' -r); do echo -n "$i - "; curl -sq https://${i} -IXGET | head -n1 ; done | uniq |
|
|
|
# linstor and kubevirt |
