Debugging ODL K8s cluster nodes notReady

gistfile1.txt

kubectl get po -A -o wide

NAMESPACE     NAME                                         READY   STATUS             RESTARTS   AGE   IP              NODE                                                   NOMINATED NODE   READINESS GATES
default       sdnc-opendaylight-0                          0/1     Pending            0          37m   <none>          <none>                                                 <none>           <none>
kube-system   calico-kube-controllers-7b67cb9dd4-vnfm8     0/1     Pending            0          44m   <none>          <none>                                                 <none>           <none>
kube-system   calico-node-9kwq4                            1/1     Running            0          40m   10.0.0.50       sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1     <none>           <none>
kube-system   calico-node-jvpgh                            1/1     Running            0          40m   10.0.0.59       sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0     <none>           <none>
kube-system   calico-node-wp484                            1/1     Running            0          44m   10.0.0.243      sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0   <none>           <none>
kube-system   coredns-57995474d5-6qbjh                     1/1     Running            0          44m   10.100.45.130   sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0   <none>           <none>
kube-system   coredns-57995474d5-xtcxb                     1/1     Running            0          44m   10.100.45.129   sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0   <none>           <none>
kube-system   csi-cinder-controllerplugin-0                5/5     Running            0          43m   10.100.45.128   sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0   <none>           <none>
kube-system   dashboard-metrics-scraper-7674b9d54f-dnqsl   0/1     Pending            0          44m   <none>          <none>                                                 <none>           <none>
kube-system   k8s-keystone-auth-8cjmz                      1/1     Running            0          43m   10.0.0.243      sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0   <none>           <none>
kube-system   kube-dns-autoscaler-7967dcdbd7-m5ct6         0/1     Pending            0          44m   <none>          <none>                                                 <none>           <none>
kube-system   kubernetes-dashboard-bfc6ccfdf-t4jl2         0/1     Pending            0          44m   <none>          <none>                                                 <none>           <none>
kube-system   openstack-cloud-controller-manager-vh77v     0/1     ImagePullBackOff   0          44m   10.0.0.243      sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0   <none>           <none>

kubectl get events --sort-by='.metadata.creationTimestamp'
LAST SEEN   TYPE      REASON                    OBJECT                                                      MESSAGE
41m         Normal    Starting                  node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0   Starting kube-proxy.
41m         Normal    RegisteredNode            node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0   Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0 event: Registered Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0 in Controller
41m         Normal    NodeReady                 node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0   Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0 status is now: NodeReady
38m         Normal    NodeAllocatableEnforced   node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1     Updated Node Allocatable limit across pods
38m         Normal    NodeHasSufficientPID      node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1     Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 status is now: NodeHasSufficientPID
38m         Normal    NodeHasNoDiskPressure     node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1     Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 status is now: NodeHasNoDiskPressure
38m         Normal    NodeHasSufficientMemory   node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1     Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 status is now: NodeHasSufficientMemory
38m         Normal    Starting                  node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1     Starting kubelet.
38m         Normal    Starting                  node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0     Starting kubelet.
38m         Normal    RegisteredNode            node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1     Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 event: Registered Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 in Controller
38m         Normal    NodeAllocatableEnforced   node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0     Updated Node Allocatable limit across pods
38m         Normal    NodeHasSufficientPID      node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0     Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 status is now: NodeHasSufficientPID
38m         Normal    NodeHasNoDiskPressure     node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0     Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 status is now: NodeHasNoDiskPressure
38m         Normal    NodeHasSufficientMemory   node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0     Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 status is now: NodeHasSufficientMemory
38m         Normal    Starting                  node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1     Starting kube-proxy.
38m         Normal    Starting                  node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0     Starting kube-proxy.
37m         Normal    RegisteredNode            node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0     Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 event: Registered Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 in Controller
37m         Normal    NodeReady                 node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1     Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1 status is now: NodeReady
37m         Normal    NodeReady                 node/sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0     Node sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0 status is now: NodeReady
34m         Warning   FailedScheduling          pod/sdnc-opendaylight-0                                     0/3 nodes are available: 1 node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate, 2 node(s) had taint {node.cloudprovider.kubernetes.io/uninitialized: true}, that the pod didn't tolerate.
34m         Normal    SuccessfulCreate          statefulset/sdnc-opendaylight                               create Pod sdnc-opendaylight-0 in StatefulSet sdnc-opendaylight successful
12m         Warning   FailedScheduling          pod/sdnc-opendaylight-0                                     0/3 nodes are available: 1 node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate, 2 node(s) had taint {node.cloudprovider.kubernetes.io/uninitialized: true}, that the pod didn't tolerate.
4m38s       Warning   FailedGetScale            horizontalpodautoscaler/sdnc-opendaylight                   deployments/scale.apps "sdnc-opendaylight" not found

kubectl get nodes --show-labels
NAME                                                   STATUS   ROLES    AGE   VERSION   LABELS
sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0   Ready    master   39m   v1.21.1   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-master-0,kubernetes.io/os=linux,magnum.openstack.org/nodegroup=default-master,magnum.openstack.org/role=master,node-role.kubernetes.io/master=
sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0     Ready    <none>   35m   v1.21.1   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-0,kubernetes.io/os=linux,magnum.openstack.org/nodegroup=default-worker,magnum.openstack.org/role=worker
sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1     Ready    <none>   35m   v1.21.1   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=sandbox-packaging-k8s-odl-depl-ycser6ofq7ic-node-1,kubernetes.io/os=linux,magnum.openstack.org/nodegroup=default-worker,magnum.openstack.org/role=worker

> kubectl describe pods
Name:           sdnc-opendaylight-0
Namespace:      default
Priority:       0
Node:           <none>
Labels:         app.kubernetes.io/instance=sdnc
                app.kubernetes.io/name=opendaylight
                controller-revision-hash=sdnc-opendaylight-5c9c85f8c4
                statefulset.kubernetes.io/pod-name=sdnc-opendaylight-0
Annotations:    kubernetes.io/psp: magnum.privileged
Status:         Pending
IP:             
IPs:            <none>
Controlled By:  StatefulSet/sdnc-opendaylight
Init Containers:
  updatevolperm:
    Image:      busybox
    Port:       <none>
    Host Port:  <none>
    Command:
      chown
      8181
      /data
    Environment:  <none>
    Mounts:
      /data from odlvol (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-st6bd (ro)
Containers:
  opendaylight:
    Image:      nexus3.opendaylight.org:10001/opendaylight/opendaylight:14.2.0
    Port:       8181/TCP
    Host Port:  0/TCP
    Command:
      bash
      -c
      bash -x /scripts/startodl.sh
    Readiness:  tcp-socket :8181 delay=30s timeout=1s period=10s #success=1 #failure=3
    Environment:
      FEATURES:         odl-restconf,odl-restconf-all
      JAVA_HOME:        /opt/openjdk-11/
      JAVA_OPTS:        -Xms512m -Xmx2048m
      EXTRA_JAVA_OPTS:  -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:ParallelGCThreads=3 -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication 
    Mounts:
      /data from odlvol (rw)
      /scripts from scripts (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-st6bd (ro)
Conditions:
  Type           Status
  PodScheduled   False 
Volumes:
  scripts:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      sdnc-opendaylight
    Optional:  false
  odlvol:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  kube-api-access-st6bd:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason            Age   From               Message
  ----     ------            ----  ----               -------
  Warning  FailedScheduling  30m   default-scheduler  0/3 nodes are available: 1 node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate, 2 node(s) had taint {node.cloudprovider.kubernetes.io/uninitialized: true}, that the pod didn't tolerate.
  Warning  FailedScheduling  30m   default-scheduler  0/3 nodes are available: 1 node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate, 2 node(s) had taint {node.cloudprovider.kubernetes.io/uninitialized: true}, that the pod didn't tolerate.

// test the setup to connect to the SDNC and list the restconf modules

> curl -p -u admin:admin --request GET 'http://127.0.0.1:8080/restconf/modules' | jq
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  4259  100  4259    0     0   166k      0 --:--:-- --:--:-- --:--:--  173k
{
  "modules": {
    "module": [
      {
        "name": "prefix-shard-configuration",
        "revision": "2017-01-10",
        "namespace": "urn:opendaylight:params:xml:ns:yang:controller:md:sal:clustering:prefix-shard-configuration"
      },
      {
        "name": "odl-general-entity",
        "revision": "2015-09-30",
        "namespace": "urn:opendaylight:params:xml:ns:yang:mdsal:core:general-entity"
      },
      {
        "name": "aaa-password-service-config",
        "revision": "2017-06-19",
        "namespace": "urn:opendaylight:aaa:password:service:config"
      },
      {
        "name": "cluster-admin",
        "revision": "2015-10-13",
        "namespace": "urn:opendaylight:params:xml:ns:yang:controller:md:sal:cluster:admin"
      },
      {
        "name": "ietf-restconf",
        "revision": "2017-01-26",
        "namespace": "urn:ietf:params:xml:ns:yang:ietf-restconf"
      },
      {
        "name": "ietf-netconf",
        "revision": "2011-06-01",
        "namespace": "urn:ietf:params:xml:ns:netconf:base:1.0",
        "feature": [
          "confirmed-commit",
          "startup",
          "rollback-on-error",
          "validate",
          "url",
          "writable-running",
          "xpath",
          "candidate"
        ]
      },
      {
        "name": "distributed-datastore-provider",
        "revision": "2014-06-12",
        "namespace": "urn:opendaylight:params:xml:ns:yang:controller:config:distributed-datastore-provider"
      },
      {
        "name": "nc-notifications",
        "revision": "2008-07-14",
        "namespace": "urn:ietf:params:xml:ns:netmod:notification"
      },
      {
        "name": "aaa-cert",
        "revision": "2015-11-26",
        "namespace": "urn:opendaylight:yang:aaa:cert"
      },
      {
        "name": "sal-remote-augment",
        "revision": "2014-07-08",
        "namespace": "urn:sal:restconf:event:subscription"
      },
      {
        "name": "ietf-netconf-monitoring",
        "revision": "2010-10-04",
        "namespace": "urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring"
      },
      {
        "name": "ietf-netconf-nmda",
        "revision": "2019-01-07",
        "namespace": "urn:ietf:params:xml:ns:yang:ietf-netconf-nmda",
        "feature": [
          "origin",
          "with-defaults"
        ]
      },
      {
        "name": "ietf-yang-metadata",
        "revision": "2016-08-05",
        "namespace": "urn:ietf:params:xml:ns:yang:ietf-yang-metadata"
      },
      {
        "name": "ietf-netconf-with-defaults",
        "revision": "2011-06-01",
        "namespace": "urn:ietf:params:xml:ns:yang:ietf-netconf-with-defaults"
      },
      {
        "name": "ietf-inet-types",
        "revision": "2013-07-15",
        "namespace": "urn:ietf:params:xml:ns:yang:ietf-inet-types"
      },
      {
        "name": "ietf-yang-library",
        "revision": "2019-01-04",
        "namespace": "urn:ietf:params:xml:ns:yang:ietf-yang-library"
      },
      {
        "name": "yang-ext",
        "revision": "2013-07-09",
        "namespace": "urn:opendaylight:yang:extension:yang-ext"
      },
      {
        "name": "ietf-yang-types",
        "revision": "2013-07-15",
        "namespace": "urn:ietf:params:xml:ns:yang:ietf-yang-types"
      },
      {
        "name": "aaa-encrypt-service-config",
        "revision": "2016-09-15",
        "namespace": "config:aaa:authn:encrypt:service:config"
      },
      {
        "name": "subscribe-to-notification",
        "revision": "2016-10-28",
        "namespace": "subscribe:to:notification"
      },
      {
        "name": "ietf-netconf-notifications",
        "revision": "2012-02-06",
        "namespace": "urn:ietf:params:xml:ns:yang:ietf-netconf-notifications"
      },
      {
        "name": "aaa-app-config",
        "revision": "2017-06-19",
        "namespace": "urn:opendaylight:aaa:app:config"
      },
      {
        "name": "odl-controller-cds-types",
        "revision": "2019-10-24",
        "namespace": "urn:opendaylight:params:xml:ns:yang:controller:cds:types"
      },
      {
        "name": "aaa",
        "revision": "2016-12-14",
        "namespace": "urn:opendaylight:params:xml:ns:yang:aaa"
      },
      {
        "name": "notifications",
        "revision": "2008-07-14",
        "namespace": "urn:ietf:params:xml:ns:netconf:notification:1.0"
      },
      {
        "name": "aaa-cert-mdsal",
        "revision": "2016-03-21",
        "namespace": "urn:opendaylight:yang:aaa:cert:mdsal"
      },
      {
        "name": "aaa-cert-rpc",
        "revision": "2015-12-15",
        "namespace": "urn:opendaylight:yang:aaa:cert:rpc"
      },
      {
        "name": "ietf-netconf-monitoring-extension",
        "revision": "2013-12-10",
        "namespace": "urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring-extension"
      },
      {
        "name": "ietf-origin",
        "revision": "2018-02-14",
        "namespace": "urn:ietf:params:xml:ns:yang:ietf-origin"
      },
      {
        "name": "ietf-restconf",
        "revision": "2013-10-19",
        "namespace": "urn:ietf:params:xml:ns:yang:ietf-restconf"
      },
      {
        "name": "instance-identifier-patch-module",
        "revision": "2015-11-21",
        "namespace": "instance:identifier:patch:module"
      },
      {
        "name": "sal-remote",
        "revision": "2014-01-14",
        "namespace": "urn:opendaylight:params:xml:ns:yang:controller:md:sal:remote"
      },
      {
        "name": "ietf-datastores",
        "revision": "2018-02-14",
        "namespace": "urn:ietf:params:xml:ns:yang:ietf-datastores"
      },
      {
        "name": "entity-owners",
        "revision": "2015-08-04",
        "namespace": "urn:opendaylight:params:xml:ns:yang:controller:md:sal:clustering:entity-owners"
      },
      {
        "name": "ietf-restconf-monitoring",
        "revision": "2017-01-26",
        "namespace": "urn:ietf:params:xml:ns:yang:ietf-restconf-monitoring"
      }
    ]
  }
}

kubectl --namespace default port-forward $POD_NAME 8080:$CONTAINER_PORT
# Get all worker nodes
kubectl get node --selector='!node-role.kubernetes.io/master'
# Get all namespaces
kubectl get pods --all-namespaces
# Get all events by timestamps
kubectl get events --sort-by=.metadata.creationTimestamp
JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' \
>  && kubectl get nodes -o jsonpath="$JSONPATH" | grep "Ready=True"

# kubectl get nodes -o jsonpath='{range .items[*]} {.metadata.name} {" "} {.status.conditions[?(@.type=="Ready")].status} {" "} {.spec.taints} {"\n"} {end}'
#  sandbox-packaging-k8s-odl-depl-laucrtjp4kdp-master-0   True   [{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"},{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","value":"true"}]
#   sandbox-packaging-k8s-odl-depl-laucrtjp4kdp-node-0   True   [{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","value":"true"}]
#   sandbox-packaging-k8s-odl-depl-laucrtjp4kdp-node-1   True   [{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","value":"true"}]

# Wait for pod to get ready before testing
#kubectl wait -n ${POD_NAME} --for=condition=ready pod --all