Created
December 19, 2021 04:36
-
-
Save askb/cb1bb7843e2cbdfd33043f0a9056632c to your computer and use it in GitHub Desktop.
Debugging ODL K8s cluster nodes notReady
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
kubectl get po -A -o wide | |
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES | |
default sdnc-opendaylight-0 0/1 Pending 0 37m <none> <none> <none> <none> | |
kube-system calico-kube-controllers-7b67cb9dd4-vnfm8 0/1 Pending 0 44m <none> <none> <none> <none> | |
kube-system calico-node-9kwq4 1/1 Running 0 40m 10.0.0.50 sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 <none> <none> | |
kube-system calico-node-jvpgh 1/1 Running 0 40m 10.0.0.59 sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 <none> <none> | |
kube-system calico-node-wp484 1/1 Running 0 44m 10.0.0.243 sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0 <none> <none> | |
kube-system coredns-57995474d5-6qbjh 1/1 Running 0 44m 10.100.45.130 sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0 <none> <none> | |
kube-system coredns-57995474d5-xtcxb 1/1 Running 0 44m 10.100.45.129 sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0 <none> <none> | |
kube-system csi-cinder-controllerplugin-0 5/5 Running 0 43m 10.100.45.128 sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0 <none> <none> | |
kube-system dashboard-metrics-scraper-7674b9d54f-dnqsl 0/1 Pending 0 44m <none> <none> <none> <none> | |
kube-system k8s-keystone-auth-8cjmz 1/1 Running 0 43m 10.0.0.243 sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0 <none> <none> | |
kube-system kube-dns-autoscaler-7967dcdbd7-m5ct6 0/1 Pending 0 44m <none> <none> <none> <none> | |
kube-system kubernetes-dashboard-bfc6ccfdf-t4jl2 0/1 Pending 0 44m <none> <none> <none> <none> | |
kube-system openstack-cloud-controller-manager-vh77v 0/1 ImagePullBackOff 0 44m 10.0.0.243 sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0 <none> <none> | |
kubectl get events --sort-by='.metadata.creationTimestamp' | |
LAST SEEN TYPE REASON OBJECT MESSAGE | |
41m Normal Starting node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0 Starting kube-proxy. | |
41m Normal RegisteredNode node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0 Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0 event: Registered Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0 in Controller | |
41m Normal NodeReady node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0 Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0 status is now: NodeReady | |
38m Normal NodeAllocatableEnforced node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 Updated Node Allocatable limit across pods | |
38m Normal NodeHasSufficientPID node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 status is now: NodeHasSufficientPID | |
38m Normal NodeHasNoDiskPressure node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 status is now: NodeHasNoDiskPressure | |
38m Normal NodeHasSufficientMemory node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 status is now: NodeHasSufficientMemory | |
38m Normal Starting node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 Starting kubelet. | |
38m Normal Starting node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 Starting kubelet. | |
38m Normal RegisteredNode node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 event: Registered Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 in Controller | |
38m Normal NodeAllocatableEnforced node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 Updated Node Allocatable limit across pods | |
38m Normal NodeHasSufficientPID node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 status is now: NodeHasSufficientPID | |
38m Normal NodeHasNoDiskPressure node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 status is now: NodeHasNoDiskPressure | |
38m Normal NodeHasSufficientMemory node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 status is now: NodeHasSufficientMemory | |
38m Normal Starting node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 Starting kube-proxy. | |
38m Normal Starting node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 Starting kube-proxy. | |
37m Normal RegisteredNode node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 event: Registered Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 in Controller | |
37m Normal NodeReady node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 status is now: NodeReady | |
37m Normal NodeReady node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 status is now: NodeReady | |
34m Warning FailedScheduling pod/sdnc-opendaylight-0 0/3 nodes are available: 1 node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate, 2 node(s) had taint {node.cloudprovider.kubernetes.io/uninitialized: true}, that the pod didn't tolerate. | |
34m Normal SuccessfulCreate statefulset/sdnc-opendaylight create Pod sdnc-opendaylight-0 in StatefulSet sdnc-opendaylight successful | |
12m Warning FailedScheduling pod/sdnc-opendaylight-0 0/3 nodes are available: 1 node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate, 2 node(s) had taint {node.cloudprovider.kubernetes.io/uninitialized: true}, that the pod didn't tolerate. | |
4m38s Warning FailedGetScale horizontalpodautoscaler/sdnc-opendaylight deployments/scale.apps "sdnc-opendaylight" not found | |
kubectl get nodes --show-labels | |
NAME STATUS ROLES AGE VERSION LABELS | |
sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0 Ready master 39m v1.21.1 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0,kubernetes.io/os=linux,magnum.openstack.org/nodegroup=default-master,magnum.openstack.org/role=master,node-role.kubernetes.io/master= | |
sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 Ready <none> 35m v1.21.1 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0,kubernetes.io/os=linux,magnum.openstack.org/nodegroup=default-worker,magnum.openstack.org/role=worker | |
sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 Ready <none> 35m v1.21.1 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1,kubernetes.io/os=linux,magnum.openstack.org/nodegroup=default-worker,magnum.openstack.org/role=worker | |
> kubectl describe pods | |
Name: sdnc-opendaylight-0 | |
Namespace: default | |
Priority: 0 | |
Node: <none> | |
Labels: app.kubernetes.io/instance=sdnc | |
app.kubernetes.io/name=opendaylight | |
controller-revision-hash=sdnc-opendaylight-5c9c85f8c4 | |
statefulset.kubernetes.io/pod-name=sdnc-opendaylight-0 | |
Annotations: kubernetes.io/psp: magnum.privileged | |
Status: Pending | |
IP: | |
IPs: <none> | |
Controlled By: StatefulSet/sdnc-opendaylight | |
Init Containers: | |
updatevolperm: | |
Image: busybox | |
Port: <none> | |
Host Port: <none> | |
Command: | |
chown | |
8181 | |
/data | |
Environment: <none> | |
Mounts: | |
/data from odlvol (rw) | |
/var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-st6bd (ro) | |
Containers: | |
opendaylight: | |
Image: nexus3.opendaylight.org:10001/opendaylight/opendaylight:14.2.0 | |
Port: 8181/TCP | |
Host Port: 0/TCP | |
Command: | |
bash | |
-c | |
bash -x /scripts/startodl.sh | |
Readiness: tcp-socket :8181 delay=30s timeout=1s period=10s #success=1 #failure=3 | |
Environment: | |
FEATURES: odl-restconf,odl-restconf-all | |
JAVA_HOME: /opt/openjdk-11/ | |
JAVA_OPTS: -Xms512m -Xmx2048m | |
EXTRA_JAVA_OPTS: -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:ParallelGCThreads=3 -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication | |
Mounts: | |
/data from odlvol (rw) | |
/scripts from scripts (rw) | |
/var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-st6bd (ro) | |
Conditions: | |
Type Status | |
PodScheduled False | |
Volumes: | |
scripts: | |
Type: ConfigMap (a volume populated by a ConfigMap) | |
Name: sdnc-opendaylight | |
Optional: false | |
odlvol: | |
Type: EmptyDir (a temporary directory that shares a pod's lifetime) | |
Medium: | |
SizeLimit: <unset> | |
kube-api-access-st6bd: | |
Type: Projected (a volume that contains injected data from multiple sources) | |
TokenExpirationSeconds: 3607 | |
ConfigMapName: kube-root-ca.crt | |
ConfigMapOptional: <nil> | |
DownwardAPI: true | |
QoS Class: BestEffort | |
Node-Selectors: <none> | |
Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s | |
node.kubernetes.io/unreachable:NoExecute op=Exists for 300s | |
Events: | |
Type Reason Age From Message | |
---- ------ ---- ---- ------- | |
Warning FailedScheduling 30m default-scheduler 0/3 nodes are available: 1 node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate, 2 node(s) had taint {node.cloudprovider.kubernetes.io/uninitialized: true}, that the pod didn't tolerate. | |
Warning FailedScheduling 30m default-scheduler 0/3 nodes are available: 1 node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate, 2 node(s) had taint {node.cloudprovider.kubernetes.io/uninitialized: true}, that the pod didn't tolerate. | |
// start the sdnc with port forward:
> kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.254.0.1 <none> 443/TCP 34m
sdnc-opendaylight ClusterIP None <none> 8181/TCP 27m
[jenkins@snd-centos7-helm-4c-4g-276 ~]> export POD_NAME=$(kubectl get pods --namespace default -l "app.kubernetes.io/name=opendaylight,app.kubernetes.io/instance=sdnc" -o jsonpath="{.items[0].metadata.name}")
[jenkins@snd-centos7-helm-4c-4g-276 ~]> export CONTAINER_PORT=$(kubectl get pod --namespace default $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
[jenkins@snd-centos7-helm-4c-4g-276 ~]> kubectl --namespace default port-forward $POD_NAME 8080:$CONTAINER_PORT
Forwarding from 127.0.0.1:8080 -> 8181
Forwarding from [::1]:8080 -> 8181
Handling connection for 8080
// test the setup to connect to the SDNC and list the restconf modules
> curl -p -u admin:admin --request GET 'http://127.0.0.1:8080/restconf/modules' | jq
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 4259 100 4259 0 0 166k 0 --:--:-- --:--:-- --:--:-- 173k
{
"modules": {
"module": [
{
"name": "prefix-shard-configuration",
"revision": "2017-01-10",
"namespace": "urn:opendaylight:params:xml:ns:yang:controller:md:sal:clustering:prefix-shard-configuration"
},
{
"name": "odl-general-entity",
"revision": "2015-09-30",
"namespace": "urn:opendaylight:params:xml:ns:yang:mdsal:core:general-entity"
},
{
"name": "aaa-password-service-config",
"revision": "2017-06-19",
"namespace": "urn:opendaylight:aaa:password:service:config"
},
{
"name": "cluster-admin",
"revision": "2015-10-13",
"namespace": "urn:opendaylight:params:xml:ns:yang:controller:md:sal:cluster:admin"
},
{
"name": "ietf-restconf",
"revision": "2017-01-26",
"namespace": "urn:ietf:params:xml:ns:yang:ietf-restconf"
},
{
"name": "ietf-netconf",
"revision": "2011-06-01",
"namespace": "urn:ietf:params:xml:ns:netconf:base:1.0",
"feature": [
"confirmed-commit",
"startup",
"rollback-on-error",
"validate",
"url",
"writable-running",
"xpath",
"candidate"
]
},
{
"name": "distributed-datastore-provider",
"revision": "2014-06-12",
"namespace": "urn:opendaylight:params:xml:ns:yang:controller:config:distributed-datastore-provider"
},
{
"name": "nc-notifications",
"revision": "2008-07-14",
"namespace": "urn:ietf:params:xml:ns:netmod:notification"
},
{
"name": "aaa-cert",
"revision": "2015-11-26",
"namespace": "urn:opendaylight:yang:aaa:cert"
},
{
"name": "sal-remote-augment",
"revision": "2014-07-08",
"namespace": "urn:sal:restconf:event:subscription"
},
{
"name": "ietf-netconf-monitoring",
"revision": "2010-10-04",
"namespace": "urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring"
},
{
"name": "ietf-netconf-nmda",
"revision": "2019-01-07",
"namespace": "urn:ietf:params:xml:ns:yang:ietf-netconf-nmda",
"feature": [
"origin",
"with-defaults"
]
},
{
"name": "ietf-yang-metadata",
"revision": "2016-08-05",
"namespace": "urn:ietf:params:xml:ns:yang:ietf-yang-metadata"
},
{
"name": "ietf-netconf-with-defaults",
"revision": "2011-06-01",
"namespace": "urn:ietf:params:xml:ns:yang:ietf-netconf-with-defaults"
},
{
"name": "ietf-inet-types",
"revision": "2013-07-15",
"namespace": "urn:ietf:params:xml:ns:yang:ietf-inet-types"
},
{
"name": "ietf-yang-library",
"revision": "2019-01-04",
"namespace": "urn:ietf:params:xml:ns:yang:ietf-yang-library"
},
{
"name": "yang-ext",
"revision": "2013-07-09",
"namespace": "urn:opendaylight:yang:extension:yang-ext"
},
{
"name": "ietf-yang-types",
"revision": "2013-07-15",
"namespace": "urn:ietf:params:xml:ns:yang:ietf-yang-types"
},
{
"name": "aaa-encrypt-service-config",
"revision": "2016-09-15",
"namespace": "config:aaa:authn:encrypt:service:config"
},
{
"name": "subscribe-to-notification",
"revision": "2016-10-28",
"namespace": "subscribe:to:notification"
},
{
"name": "ietf-netconf-notifications",
"revision": "2012-02-06",
"namespace": "urn:ietf:params:xml:ns:yang:ietf-netconf-notifications"
},
{
"name": "aaa-app-config",
"revision": "2017-06-19",
"namespace": "urn:opendaylight:aaa:app:config"
},
{
"name": "odl-controller-cds-types",
"revision": "2019-10-24",
"namespace": "urn:opendaylight:params:xml:ns:yang:controller:cds:types"
},
{
"name": "aaa",
"revision": "2016-12-14",
"namespace": "urn:opendaylight:params:xml:ns:yang:aaa"
},
{
"name": "notifications",
"revision": "2008-07-14",
"namespace": "urn:ietf:params:xml:ns:netconf:notification:1.0"
},
{
"name": "aaa-cert-mdsal",
"revision": "2016-03-21",
"namespace": "urn:opendaylight:yang:aaa:cert:mdsal"
},
{
"name": "aaa-cert-rpc",
"revision": "2015-12-15",
"namespace": "urn:opendaylight:yang:aaa:cert:rpc"
},
{
"name": "ietf-netconf-monitoring-extension",
"revision": "2013-12-10",
"namespace": "urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring-extension"
},
{
"name": "ietf-origin",
"revision": "2018-02-14",
"namespace": "urn:ietf:params:xml:ns:yang:ietf-origin"
},
{
"name": "ietf-restconf",
"revision": "2013-10-19",
"namespace": "urn:ietf:params:xml:ns:yang:ietf-restconf"
},
{
"name": "instance-identifier-patch-module",
"revision": "2015-11-21",
"namespace": "instance:identifier:patch:module"
},
{
"name": "sal-remote",
"revision": "2014-01-14",
"namespace": "urn:opendaylight:params:xml:ns:yang:controller:md:sal:remote"
},
{
"name": "ietf-datastores",
"revision": "2018-02-14",
"namespace": "urn:ietf:params:xml:ns:yang:ietf-datastores"
},
{
"name": "entity-owners",
"revision": "2015-08-04",
"namespace": "urn:opendaylight:params:xml:ns:yang:controller:md:sal:clustering:entity-owners"
},
{
"name": "ietf-restconf-monitoring",
"revision": "2017-01-26",
"namespace": "urn:ietf:params:xml:ns:yang:ietf-restconf-monitoring"
}
]
}
}
kubectl --namespace default port-forward $POD_NAME 8080:$CONTAINER_PORT
# Get all worker nodes
kubectl get node --selector='!node-role.kubernetes.io/master'
# Get all namespaces
kubectl get pods --all-namespaces
# Get all events by timestamps
kubectl get events --sort-by=.metadata.creationTimestamp
JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' \
> && kubectl get nodes -o jsonpath="$JSONPATH" | grep "Ready=True"
# kubectl get nodes -o jsonpath='{range .items[*]} {.metadata.name} {" "} {.status.conditions[?(@.type=="Ready")].status} {" "} {.spec.taints} {"\n"} {end}'
# sandbox-packaging-k8s-odl-depl-laucrtjp4kdp-master-0 True [{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"},{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","value":"true"}]
# sandbox-packaging-k8s-odl-depl-laucrtjp4kdp-node-0 True [{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","value":"true"}]
# sandbox-packaging-k8s-odl-depl-laucrtjp4kdp-node-1 True [{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","value":"true"}]
# Wait for pod to get ready before testing
#kubectl wait -n ${POD_NAME} --for=condition=ready pod --all
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
// after fixing this we can see that the pod gets rebooted: