Skip to content

Instantly share code, notes, and snippets.

@nerdalert

nerdalert/gist:451896250569e0852e28a907a35198d7 Secret

Created June 9, 2020 20:26
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save nerdalert/451896250569e0852e28a907a35198d7 to your computer and use it in GitHub Desktop.
Save nerdalert/451896250569e0852e28a907a35198d7 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
gistfile1.txt
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
kube-system coredns-66bff467f8-9vnf5 0/1 Pending 0 41m <none> <none> <none> <none>
kube-system coredns-66bff467f8-wknqs 0/1 Pending 0 41m <none> <none> <none> <none>
kube-system etcd-ovn-control-plane 1/1 Running 0 42m 172.17.0.3 ovn-control-plane <none> <none>
kube-system kube-apiserver-ovn-control-plane 1/1 Running 0 42m 172.17.0.3 ovn-control-plane <none> <none>
kube-system kube-controller-manager-ovn-control-plane 1/1 Running 0 42m 172.17.0.3 ovn-control-plane <none> <none>
kube-system kube-proxy-kzll6 1/1 Running 0 41m 172.17.0.3 ovn-control-plane <none> <none>
kube-system kube-proxy-q5l5z 1/1 Running 0 41m 172.17.0.4 ovn-worker2 <none> <none>
kube-system kube-proxy-zbnq4 1/1 Running 0 41m 172.17.0.2 ovn-worker <none> <none>
kube-system kube-scheduler-ovn-control-plane 1/1 Running 0 42m 172.17.0.3 ovn-control-plane <none> <none>
local-path-storage local-path-provisioner-bd4bb6b75-ll9t6 0/1 Pending 0 41m <none> <none> <none> <none>
openshift-multus multus-42bkd 1/1 Running 2 35m 172.17.0.2 ovn-worker <none> <none>
openshift-multus multus-pgq9v 1/1 Running 2 35m 172.17.0.4 ovn-worker2 <none> <none>
openshift-multus multus-ztjjj 1/1 Running 2 35m 172.17.0.3 ovn-control-plane <none> <none>
openshift-network-operator network-operator-68f68bc9bc-rm64n 1/1 Running 0 36m 172.17.0.3 ovn-control-plane <none> <none>
openshift-ovn-kubernetes ovnkube-master-8khjb 3/4 CrashLoopBackOff 11 35m 172.17.0.3 ovn-control-plane <none> <none>
openshift-ovn-kubernetes ovnkube-node-h89h5 1/2 CrashLoopBackOff 6 35m 172.17.0.2 ovn-worker <none> <none>
openshift-ovn-kubernetes ovnkube-node-krhdv 1/2 Running 7 35m 172.17.0.3 ovn-control-plane <none> <none>
openshift-ovn-kubernetes ovnkube-node-xtblf 1/2 Running 7 35m 172.17.0.4 ovn-worker2 <none> <none>
openshift-ovn-kubernetes ovs-node-dgsbq 1/1 Running 0 35m 172.17.0.3 ovn-control-plane <none> <none>
openshift-ovn-kubernetes ovs-node-jgdrx 1/1 Running 0 35m 172.17.0.4 ovn-worker2 <none> <none>
openshift-ovn-kubernetes ovs-node-nvnks 1/1 Running 0 35m 172.17.0.2 ovn-worker <none> <none>
$kubectl describe pod ovnkube-master-8khjb -n openshift-ovn-kubernetes
Name: ovnkube-master-8khjb
Namespace: openshift-ovn-kubernetes
Priority: 2000000000
Priority Class Name: system-cluster-critical
Node: ovn-control-plane/172.17.0.3
Start Time: Tue, 09 Jun 2020 15:45:50 -0400
Labels: app=ovnkube-master
component=network
controller-revision-hash=64dc865b55
kubernetes.io/os=linux
openshift.io/component=network
pod-template-generation=1
type=infra
Annotations: <none>
Status: Running
IP: 172.17.0.3
IPs:
IP: 172.17.0.3
Controlled By: DaemonSet/ovnkube-master
Containers:
northd:
Container ID: containerd://e3290bfd09cad7b8f226f9c86b157815c6ddaf4ef5c55771e5629878398307d8
Image: origin-ovn-kubernetes:dev
Image ID: sha256:05d261326bef5dfcf4e985361c941faf9292fb4ffbd920c72fddc86c4ded7603
Port: <none>
Host Port: <none>
Command:
/bin/bash
-c
set -xe
if [[ -f /env/_master ]]; then
set -o allexport
source /env/_master
set +o allexport
fi
exec ovn-northd \
--no-chdir "-vconsole:${OVN_LOG_LEVEL}" -vfile:off \
--ovnnb-db "ssl:172.17.0.3:9641" \
--ovnsb-db "ssl:172.17.0.3:9642" \
-p /ovn-cert/tls.key \
-c /ovn-cert/tls.crt \
-C /ovn-ca/ca-bundle.crt
State: Running
Started: Tue, 09 Jun 2020 15:45:52 -0400
Ready: True
Restart Count: 0
Requests:
cpu: 100m
memory: 300Mi
Environment:
OVN_LOG_LEVEL: info
Mounts:
/env from env-overrides (rw)
/etc/openvswitch/ from etc-openvswitch (rw)
/ovn-ca from ovn-ca (rw)
/ovn-cert from ovn-cert (rw)
/run/openvswitch/ from run-openvswitch (rw)
/run/ovn/ from run-ovn (rw)
/var/lib/openvswitch/ from var-lib-openvswitch (rw)
/var/run/secrets/kubernetes.io/serviceaccount from ovn-kubernetes-controller-token-5dqm4 (ro)
nbdb:
Container ID: containerd://73cd48ec80769e139a234d58348346886fdcdcabb9e5fb6950c260cec23a7305
Image: origin-ovn-kubernetes:dev
Image ID: sha256:05d261326bef5dfcf4e985361c941faf9292fb4ffbd920c72fddc86c4ded7603
Ports: 9641/TCP, 9643/TCP
Host Ports: 9641/TCP, 9643/TCP
Command:
/bin/bash
-c
set -xe
if [[ -f /env/_master ]]; then
set -o allexport
source /env/_master
set +o allexport
fi
# Determine the ovn rundir.
if [[ -f /usr/bin/ovn-appctl ]] ; then
# ovn-appctl is present. Use new ovn run dir path.
OVNCTL_PATH=/usr/share/ovn/scripts/ovn-ctl
else
# ovn-appctl is not present. Use openvswitch run dir path.
OVNCTL_PATH=/usr/share/openvswitch/scripts/ovn-ctl
fi
bracketify() { case "$1" in *:*) echo "[$1]" ;; *) echo "$1" ;; esac }
MASTER_IP="172.17.0.3"
if [[ "${K8S_NODE_IP}" == "${MASTER_IP}" ]]; then
exec ${OVNCTL_PATH} \
--db-nb-cluster-local-port=9643 \
--db-nb-cluster-local-addr=$(bracketify ${K8S_NODE_IP}) \
--no-monitor \
--db-nb-cluster-local-proto=ssl \
--ovn-nb-db-ssl-key=/ovn-cert/tls.key \
--ovn-nb-db-ssl-cert=/ovn-cert/tls.crt \
--ovn-nb-db-ssl-ca-cert=/ovn-ca/ca-bundle.crt \
--ovn-nb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off" \
run_nb_ovsdb
else
exec ${OVNCTL_PATH} \
--db-nb-cluster-local-port=9643 \
--db-nb-cluster-remote-port=9643 \
--db-nb-cluster-local-addr=$(bracketify ${K8S_NODE_IP}) \
--db-nb-cluster-remote-addr=$(bracketify ${MASTER_IP}) \
--no-monitor \
--db-nb-cluster-local-proto=ssl \
--db-nb-cluster-remote-proto=ssl \
--ovn-nb-db-ssl-key=/ovn-cert/tls.key \
--ovn-nb-db-ssl-cert=/ovn-cert/tls.crt \
--ovn-nb-db-ssl-ca-cert=/ovn-ca/ca-bundle.crt \
--ovn-nb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off" \
run_nb_ovsdb
fi
State: Running
Started: Tue, 09 Jun 2020 15:45:53 -0400
Ready: True
Restart Count: 0
Requests:
cpu: 100m
memory: 300Mi
Readiness: exec [/bin/bash -c set -xe
# Determine the ovn rundir.
if [[ -f /usr/bin/ovn-appctl ]] ; then
# ovn-appctl is present. Use new ovn run dir path.
DB_SOCK_PATH=/var/run/ovn/ovnnb_db.ctl
APPCTL_PATH=/usr/bin/ovn-appctl
else
# ovn-appctl is not present. Use openvswitch run dir path.
DB_SOCK_PATH=/var/run/openvswitch/ovnnb_db.ctl
APPCTL_PATH=/usr/bin/ovs-appctl
fi
exec ${APPCTL_PATH} -t ${DB_SOCK_PATH} cluster/status OVN_Northbound 2>/dev/null | grep ${K8S_NODE_IP} | grep -v Address -q
] delay=30s timeout=1s period=10s #success=1 #failure=3
Environment:
OVN_LOG_LEVEL: info
K8S_NODE_IP: (v1:status.hostIP)
Mounts:
/env from env-overrides (rw)
/etc/openvswitch/ from etc-openvswitch (rw)
/etc/ovn/ from etc-openvswitch (rw)
/ovn-ca from ovn-ca (rw)
/ovn-cert from ovn-cert (rw)
/run/openvswitch/ from run-openvswitch (rw)
/run/ovn/ from run-ovn (rw)
/var/lib/openvswitch/ from var-lib-openvswitch (rw)
/var/run/secrets/kubernetes.io/serviceaccount from ovn-kubernetes-controller-token-5dqm4 (ro)
sbdb:
Container ID: containerd://d741b146e480d37435a1b48dffb92580f69c267faf8dccb0ff61f14f3d522cc7
Image: origin-ovn-kubernetes:dev
Image ID: sha256:05d261326bef5dfcf4e985361c941faf9292fb4ffbd920c72fddc86c4ded7603
Ports: 9642/TCP, 9644/TCP
Host Ports: 9642/TCP, 9644/TCP
Command:
/bin/bash
-c
set -xe
if [[ -f /env/_master ]]; then
set -o allexport
source /env/_master
set +o allexport
fi
# Determine the ovn rundir.
if [[ -f /usr/bin/ovn-appctl ]] ; then
# ovn-appctl is present. Use new ovn run dir path.
OVNCTL_PATH=/usr/share/ovn/scripts/ovn-ctl
else
# ovn-appctl is not present. Use openvswitch run dir path.
OVNCTL_PATH=/usr/share/openvswitch/scripts/ovn-ctl
fi
bracketify() { case "$1" in *:*) echo "[$1]" ;; *) echo "$1" ;; esac }
MASTER_IP="172.17.0.3"
if [[ "${K8S_NODE_IP}" == "${MASTER_IP}" ]]; then
exec ${OVNCTL_PATH} \
--db-sb-cluster-local-port=9644 \
--db-sb-cluster-local-addr=$(bracketify ${K8S_NODE_IP}) \
--no-monitor \
--db-sb-cluster-local-proto=ssl \
--ovn-sb-db-ssl-key=/ovn-cert/tls.key \
--ovn-sb-db-ssl-cert=/ovn-cert/tls.crt \
--ovn-sb-db-ssl-ca-cert=/ovn-ca/ca-bundle.crt \
--ovn-sb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off" \
run_sb_ovsdb
else
echo "joining cluster at ${MASTER_IP}"
exec ${OVNCTL_PATH} \
--db-sb-cluster-local-port=9644 \
--db-sb-cluster-remote-port=9644 \
--db-sb-cluster-local-addr=$(bracketify ${K8S_NODE_IP}) \
--db-sb-cluster-remote-addr=$(bracketify ${MASTER_IP}) \
--no-monitor \
--db-sb-cluster-local-proto=ssl \
--db-sb-cluster-remote-proto=ssl \
--ovn-sb-db-ssl-key=/ovn-cert/tls.key \
--ovn-sb-db-ssl-cert=/ovn-cert/tls.crt \
--ovn-sb-db-ssl-ca-cert=/ovn-ca/ca-bundle.crt \
--ovn-sb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off" \
run_sb_ovsdb
fi
State: Running
Started: Tue, 09 Jun 2020 15:45:55 -0400
Ready: True
Restart Count: 0
Readiness: exec [/bin/bash -c set -xe
# Determine the ovn rundir.
if [[ -f /usr/bin/ovn-appctl ]] ; then
# ovn-appctl is present. Use new ovn run dir path.
DB_SOCK_PATH=/var/run/ovn/ovnsb_db.ctl
APPCTL_PATH=/usr/bin/ovn-appctl
else
# ovn-appctl is not present. Use openvswitch run dir path.
DB_SOCK_PATH=/var/run/openvswitch/ovnsb_db.ctl
APPCTL_PATH=/usr/bin/ovs-appctl
fi
exec ${APPCTL_PATH} -t ${DB_SOCK_PATH} cluster/status OVN_Southbound 2>/dev/null | grep ${K8S_NODE_IP} | grep -v Address -q
] delay=30s timeout=1s period=10s #success=1 #failure=3
Environment:
OVN_LOG_LEVEL: info
K8S_NODE_IP: (v1:status.hostIP)
Mounts:
/env from env-overrides (rw)
/etc/openvswitch/ from etc-openvswitch (rw)
/etc/ovn/ from etc-openvswitch (rw)
/ovn-ca from ovn-ca (rw)
/ovn-cert from ovn-cert (rw)
/run/openvswitch/ from run-openvswitch (rw)
/run/ovn/ from run-ovn (rw)
/var/lib/openvswitch/ from var-lib-openvswitch (rw)
/var/run/secrets/kubernetes.io/serviceaccount from ovn-kubernetes-controller-token-5dqm4 (ro)
ovnkube-master:
Container ID: containerd://cb56c6fb6da7f4a9f04455493299b421e8c645b97f3a86b8c82beb50463d682a
Image: origin-ovn-kubernetes:dev
Image ID: sha256:05d261326bef5dfcf4e985361c941faf9292fb4ffbd920c72fddc86c4ded7603
Port: 9102/TCP
Host Port: 9102/TCP
Command:
/bin/bash
-c
set -xe
if [[ -f "/env/_master" ]]; then
set -o allexport
source "/env/_master"
set +o allexport
fi
hybrid_overlay_flags=
if [[ -n "" ]]; then
hybrid_overlay_flags="--enable-hybrid-overlay"
if [[ -n "" ]]; then
hybrid_overlay_flags="${hybrid_overlay_flags} --hybrid-overlay-cluster-subnets="
fi
fi
# start nbctl daemon for caching
export OVN_NB_DAEMON=$(ovn-nbctl --pidfile=/tmp/ovnk-nbctl.pid \
--detach \
-p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt \
--db "ssl:172.17.0.3:9641")
exec /usr/bin/ovnkube \
--init-master "${K8S_NODE}" \
--config-file=/run/ovnkube-config/ovnkube.conf \
--ovn-empty-lb-events \
--loglevel "${OVN_KUBE_LOG_LEVEL}" \
${hybrid_overlay_flags} \
--metrics-bind-address "0.0.0.0:9102" \
--sb-address "ssl://172.17.0.3:9642" \
--sb-client-privkey /ovn-cert/tls.key \
--sb-client-cert /ovn-cert/tls.crt \
--sb-client-cacert /ovn-ca/ca-bundle.crt
State: Waiting
Reason: CrashLoopBackOff
Last State: Terminated
Reason: Error
Message: 20:17:31.452738 1 shared_informer.go:227] caches populated
I0609 20:17:31.452741 1 shared_informer.go:227] caches populated
I0609 20:17:31.452745 1 shared_informer.go:227] caches populated
I0609 20:17:31.452749 1 shared_informer.go:227] caches populated
I0609 20:17:31.452840 1 ovnkube.go:315] Watching config file /run/ovnkube-config/ovnkube.conf for changes
I0609 20:17:31.452886 1 ovnkube.go:315] Watching config file /run/ovnkube-config/..2020_06_09_19_45_50.440900408/ovnkube.conf for changes
I0609 20:17:31.457368 1 leaderelection.go:242] attempting to acquire leader lease openshift-ovn-kubernetes/ovn-kubernetes-master...
I0609 20:17:31.464401 1 leaderelection.go:252] successfully acquired lease openshift-ovn-kubernetes/ovn-kubernetes-master
I0609 20:17:31.464562 1 master.go:57] won leader election; in active mode
E0609 20:17:31.487560 1 ovs.go:668] Failed to query OVN NB DB for SCTP support, stdout: "", stderr: "ovsdb-client: failed to connect to \"unix:/var/run/openvswitch/db.sock\" (No such file or directory)\n", error: OVN command '/usr/bin/ovsdb-client list-columns --data=bare --no-heading --format=json OVN_Northbound Load_Balancer' failed: exit status 1
E0609 20:17:31.487584 1 master.go:163] Failed to setup master (OVN command '/usr/bin/ovsdb-client list-columns --data=bare --no-heading --format=json OVN_Northbound Load_Balancer' failed: exit status 1)
panic: OVN command '/usr/bin/ovsdb-client list-columns --data=bare --no-heading --format=json OVN_Northbound Load_Balancer' failed: exit status 1
goroutine 159 [running]:
github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn.(*Controller).Start.func1(0x19cd5c0, 0xc0001ac2c0)
/home/brent/code/go/src/github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/master.go:65 +0x1d9
created by k8s.io/client-go/tools/leaderelection.(*LeaderElector).Run
/home/brent/code/go/src/github.com/ovn-org/ovn-kubernetes/go-controller/vendor/k8s.io/client-go/tools/leaderelection/leaderelection.go:207 +0x132
Exit Code: 2
Started: Tue, 09 Jun 2020 16:17:31 -0400
Finished: Tue, 09 Jun 2020 16:17:31 -0400
Ready: False
Restart Count: 11
Requests:
cpu: 100m
memory: 300Mi
Environment:
OVN_KUBE_LOG_LEVEL: 4
K8S_NODE: (v1:spec.nodeName)
Mounts:
/env from env-overrides (rw)
/etc/openvswitch/ from etc-openvswitch (rw)
/etc/ovn/ from etc-openvswitch (rw)
/ovn-ca from ovn-ca (rw)
/ovn-cert from ovn-cert (rw)
/run/openvswitch/ from run-openvswitch (rw)
/run/ovn/ from run-ovn (rw)
/run/ovnkube-config/ from ovnkube-config (rw)
/var/lib/openvswitch/ from var-lib-openvswitch (rw)
/var/run/secrets/kubernetes.io/serviceaccount from ovn-kubernetes-controller-token-5dqm4 (ro)
Conditions:
Type Status
Initialized True
Ready False
ContainersReady False
PodScheduled True
Volumes:
etc-openvswitch:
Type: HostPath (bare host directory volume)
Path: /var/lib/ovn/etc
HostPathType:
var-lib-openvswitch:
Type: HostPath (bare host directory volume)
Path: /var/lib/ovn/data
HostPathType:
run-openvswitch:
Type: EmptyDir (a temporary directory that shares a pod's lifetime)
Medium:
SizeLimit: <unset>
run-ovn:
Type: HostPath (bare host directory volume)
Path: /var/run/ovn
HostPathType:
ovnkube-config:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: ovnkube-config
Optional: false
env-overrides:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: env-overrides
Optional: true
ovn-ca:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: ovn-ca
Optional: false
ovn-cert:
Type: Secret (a volume populated by a Secret)
SecretName: ovn-cert
Optional: false
ovn-kubernetes-controller-token-5dqm4:
Type: Secret (a volume populated by a Secret)
SecretName: ovn-kubernetes-controller-token-5dqm4
Optional: false
QoS Class: Burstable
Node-Selectors: beta.kubernetes.io/os=linux
node-role.kubernetes.io/master=
Tolerations: node-role.kubernetes.io/master
node.kubernetes.io/disk-pressure:NoSchedule
node.kubernetes.io/memory-pressure:NoSchedule
node.kubernetes.io/network-unavailable
node.kubernetes.io/network-unavailable:NoSchedule
node.kubernetes.io/not-ready
node.kubernetes.io/not-ready:NoExecute
node.kubernetes.io/pid-pressure:NoSchedule
node.kubernetes.io/unreachable:NoExecute
node.kubernetes.io/unreachable
node.kubernetes.io/unschedulable:NoSchedule
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 32m default-scheduler Successfully assigned openshift-ovn-kubernetes/ovnkube-master-8khjb to ovn-control-plane
Normal Pulled 32m kubelet, ovn-control-plane Container image "origin-ovn-kubernetes:dev" already present on machine
Normal Created 32m kubelet, ovn-control-plane Created container northd
Normal Started 32m kubelet, ovn-control-plane Started container northd
Normal Pulled 32m kubelet, ovn-control-plane Container image "origin-ovn-kubernetes:dev" already present on machine
Normal Created 32m kubelet, ovn-control-plane Created container nbdb
Normal Started 32m kubelet, ovn-control-plane Started container nbdb
Normal Pulled 32m kubelet, ovn-control-plane Container image "origin-ovn-kubernetes:dev" already present on machine
Normal Created 32m kubelet, ovn-control-plane Created container sbdb
Normal Started 32m kubelet, ovn-control-plane Started container sbdb
Normal Pulled 31m (x4 over 32m) kubelet, ovn-control-plane Container image "origin-ovn-kubernetes:dev" already present on machine
Normal Created 31m (x4 over 32m) kubelet, ovn-control-plane Created container ovnkube-master
Normal Started 31m (x4 over 32m) kubelet, ovn-control-plane Started container ovnkube-master
Warning BackOff 2m44s (x141 over 32m) kubelet, ovn-control-plane Back-off restarting failed container
(='o'=)[brent@centos8-160:hack$]$ ku describe pod ovnkube-master-8khjb -n openshift-ovn-kubernetes
Name: ovnkube-master-8khjb
Namespace: openshift-ovn-kubernetes
Priority: 2000000000
Priority Class Name: system-cluster-critical
Node: ovn-control-plane/172.17.0.3
Start Time: Tue, 09 Jun 2020 15:45:50 -0400
Labels: app=ovnkube-master
component=network
controller-revision-hash=64dc865b55
kubernetes.io/os=linux
openshift.io/component=network
pod-template-generation=1
type=infra
Annotations: <none>
Status: Running
IP: 172.17.0.3
IPs:
IP: 172.17.0.3
Controlled By: DaemonSet/ovnkube-master
Containers:
northd:
Container ID: containerd://e3290bfd09cad7b8f226f9c86b157815c6ddaf4ef5c55771e5629878398307d8
Image: origin-ovn-kubernetes:dev
Image ID: sha256:05d261326bef5dfcf4e985361c941faf9292fb4ffbd920c72fddc86c4ded7603
Port: <none>
Host Port: <none>
Command:
/bin/bash
-c
set -xe
if [[ -f /env/_master ]]; then
set -o allexport
source /env/_master
set +o allexport
fi
exec ovn-northd \
--no-chdir "-vconsole:${OVN_LOG_LEVEL}" -vfile:off \
--ovnnb-db "ssl:172.17.0.3:9641" \
--ovnsb-db "ssl:172.17.0.3:9642" \
-p /ovn-cert/tls.key \
-c /ovn-cert/tls.crt \
-C /ovn-ca/ca-bundle.crt
State: Running
Started: Tue, 09 Jun 2020 15:45:52 -0400
Ready: True
Restart Count: 0
Requests:
cpu: 100m
memory: 300Mi
Environment:
OVN_LOG_LEVEL: info
Mounts:
/env from env-overrides (rw)
/etc/openvswitch/ from etc-openvswitch (rw)
/ovn-ca from ovn-ca (rw)
/ovn-cert from ovn-cert (rw)
/run/openvswitch/ from run-openvswitch (rw)
/run/ovn/ from run-ovn (rw)
/var/lib/openvswitch/ from var-lib-openvswitch (rw)
/var/run/secrets/kubernetes.io/serviceaccount from ovn-kubernetes-controller-token-5dqm4 (ro)
nbdb:
Container ID: containerd://73cd48ec80769e139a234d58348346886fdcdcabb9e5fb6950c260cec23a7305
Image: origin-ovn-kubernetes:dev
Image ID: sha256:05d261326bef5dfcf4e985361c941faf9292fb4ffbd920c72fddc86c4ded7603
Ports: 9641/TCP, 9643/TCP
Host Ports: 9641/TCP, 9643/TCP
Command:
/bin/bash
-c
set -xe
if [[ -f /env/_master ]]; then
set -o allexport
source /env/_master
set +o allexport
fi
# Determine the ovn rundir.
if [[ -f /usr/bin/ovn-appctl ]] ; then
# ovn-appctl is present. Use new ovn run dir path.
OVNCTL_PATH=/usr/share/ovn/scripts/ovn-ctl
else
# ovn-appctl is not present. Use openvswitch run dir path.
OVNCTL_PATH=/usr/share/openvswitch/scripts/ovn-ctl
fi
bracketify() { case "$1" in *:*) echo "[$1]" ;; *) echo "$1" ;; esac }
MASTER_IP="172.17.0.3"
if [[ "${K8S_NODE_IP}" == "${MASTER_IP}" ]]; then
exec ${OVNCTL_PATH} \
--db-nb-cluster-local-port=9643 \
--db-nb-cluster-local-addr=$(bracketify ${K8S_NODE_IP}) \
--no-monitor \
--db-nb-cluster-local-proto=ssl \
--ovn-nb-db-ssl-key=/ovn-cert/tls.key \
--ovn-nb-db-ssl-cert=/ovn-cert/tls.crt \
--ovn-nb-db-ssl-ca-cert=/ovn-ca/ca-bundle.crt \
--ovn-nb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off" \
run_nb_ovsdb
else
exec ${OVNCTL_PATH} \
--db-nb-cluster-local-port=9643 \
--db-nb-cluster-remote-port=9643 \
--db-nb-cluster-local-addr=$(bracketify ${K8S_NODE_IP}) \
--db-nb-cluster-remote-addr=$(bracketify ${MASTER_IP}) \
--no-monitor \
--db-nb-cluster-local-proto=ssl \
--db-nb-cluster-remote-proto=ssl \
--ovn-nb-db-ssl-key=/ovn-cert/tls.key \
--ovn-nb-db-ssl-cert=/ovn-cert/tls.crt \
--ovn-nb-db-ssl-ca-cert=/ovn-ca/ca-bundle.crt \
--ovn-nb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off" \
run_nb_ovsdb
fi
State: Running
Started: Tue, 09 Jun 2020 15:45:53 -0400
Ready: True
Restart Count: 0
Requests:
cpu: 100m
memory: 300Mi
Readiness: exec [/bin/bash -c set -xe
# Determine the ovn rundir.
if [[ -f /usr/bin/ovn-appctl ]] ; then
# ovn-appctl is present. Use new ovn run dir path.
DB_SOCK_PATH=/var/run/ovn/ovnnb_db.ctl
APPCTL_PATH=/usr/bin/ovn-appctl
else
# ovn-appctl is not present. Use openvswitch run dir path.
DB_SOCK_PATH=/var/run/openvswitch/ovnnb_db.ctl
APPCTL_PATH=/usr/bin/ovs-appctl
fi
exec ${APPCTL_PATH} -t ${DB_SOCK_PATH} cluster/status OVN_Northbound 2>/dev/null | grep ${K8S_NODE_IP} | grep -v Address -q
] delay=30s timeout=1s period=10s #success=1 #failure=3
Environment:
OVN_LOG_LEVEL: info
K8S_NODE_IP: (v1:status.hostIP)
Mounts:
/env from env-overrides (rw)
/etc/openvswitch/ from etc-openvswitch (rw)
/etc/ovn/ from etc-openvswitch (rw)
/ovn-ca from ovn-ca (rw)
/ovn-cert from ovn-cert (rw)
/run/openvswitch/ from run-openvswitch (rw)
/run/ovn/ from run-ovn (rw)
/var/lib/openvswitch/ from var-lib-openvswitch (rw)
/var/run/secrets/kubernetes.io/serviceaccount from ovn-kubernetes-controller-token-5dqm4 (ro)
sbdb:
Container ID: containerd://d741b146e480d37435a1b48dffb92580f69c267faf8dccb0ff61f14f3d522cc7
Image: origin-ovn-kubernetes:dev
Image ID: sha256:05d261326bef5dfcf4e985361c941faf9292fb4ffbd920c72fddc86c4ded7603
Ports: 9642/TCP, 9644/TCP
Host Ports: 9642/TCP, 9644/TCP
Command:
/bin/bash
-c
set -xe
if [[ -f /env/_master ]]; then
set -o allexport
source /env/_master
set +o allexport
fi
# Determine the ovn rundir.
if [[ -f /usr/bin/ovn-appctl ]] ; then
# ovn-appctl is present. Use new ovn run dir path.
OVNCTL_PATH=/usr/share/ovn/scripts/ovn-ctl
else
# ovn-appctl is not present. Use openvswitch run dir path.
OVNCTL_PATH=/usr/share/openvswitch/scripts/ovn-ctl
fi
bracketify() { case "$1" in *:*) echo "[$1]" ;; *) echo "$1" ;; esac }
MASTER_IP="172.17.0.3"
if [[ "${K8S_NODE_IP}" == "${MASTER_IP}" ]]; then
exec ${OVNCTL_PATH} \
--db-sb-cluster-local-port=9644 \
--db-sb-cluster-local-addr=$(bracketify ${K8S_NODE_IP}) \
--no-monitor \
--db-sb-cluster-local-proto=ssl \
--ovn-sb-db-ssl-key=/ovn-cert/tls.key \
--ovn-sb-db-ssl-cert=/ovn-cert/tls.crt \
--ovn-sb-db-ssl-ca-cert=/ovn-ca/ca-bundle.crt \
--ovn-sb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off" \
run_sb_ovsdb
else
echo "joining cluster at ${MASTER_IP}"
exec ${OVNCTL_PATH} \
--db-sb-cluster-local-port=9644 \
--db-sb-cluster-remote-port=9644 \
--db-sb-cluster-local-addr=$(bracketify ${K8S_NODE_IP}) \
--db-sb-cluster-remote-addr=$(bracketify ${MASTER_IP}) \
--no-monitor \
--db-sb-cluster-local-proto=ssl \
--db-sb-cluster-remote-proto=ssl \
--ovn-sb-db-ssl-key=/ovn-cert/tls.key \
--ovn-sb-db-ssl-cert=/ovn-cert/tls.crt \
--ovn-sb-db-ssl-ca-cert=/ovn-ca/ca-bundle.crt \
--ovn-sb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off" \
run_sb_ovsdb
fi
State: Running
Started: Tue, 09 Jun 2020 15:45:55 -0400
Ready: True
Restart Count: 0
Readiness: exec [/bin/bash -c set -xe
# Determine the ovn rundir.
if [[ -f /usr/bin/ovn-appctl ]] ; then
# ovn-appctl is present. Use new ovn run dir path.
DB_SOCK_PATH=/var/run/ovn/ovnsb_db.ctl
APPCTL_PATH=/usr/bin/ovn-appctl
else
# ovn-appctl is not present. Use openvswitch run dir path.
DB_SOCK_PATH=/var/run/openvswitch/ovnsb_db.ctl
APPCTL_PATH=/usr/bin/ovs-appctl
fi
exec ${APPCTL_PATH} -t ${DB_SOCK_PATH} cluster/status OVN_Southbound 2>/dev/null | grep ${K8S_NODE_IP} | grep -v Address -q
] delay=30s timeout=1s period=10s #success=1 #failure=3
Environment:
OVN_LOG_LEVEL: info
K8S_NODE_IP: (v1:status.hostIP)
Mounts:
/env from env-overrides (rw)
/etc/openvswitch/ from etc-openvswitch (rw)
/etc/ovn/ from etc-openvswitch (rw)
/ovn-ca from ovn-ca (rw)
/ovn-cert from ovn-cert (rw)
/run/openvswitch/ from run-openvswitch (rw)
/run/ovn/ from run-ovn (rw)
/var/lib/openvswitch/ from var-lib-openvswitch (rw)
/var/run/secrets/kubernetes.io/serviceaccount from ovn-kubernetes-controller-token-5dqm4 (ro)
ovnkube-master:
Container ID: containerd://cb56c6fb6da7f4a9f04455493299b421e8c645b97f3a86b8c82beb50463d682a
Image: origin-ovn-kubernetes:dev
Image ID: sha256:05d261326bef5dfcf4e985361c941faf9292fb4ffbd920c72fddc86c4ded7603
Port: 9102/TCP
Host Port: 9102/TCP
Command:
/bin/bash
-c
set -xe
if [[ -f "/env/_master" ]]; then
set -o allexport
source "/env/_master"
set +o allexport
fi
hybrid_overlay_flags=
if [[ -n "" ]]; then
hybrid_overlay_flags="--enable-hybrid-overlay"
if [[ -n "" ]]; then
hybrid_overlay_flags="${hybrid_overlay_flags} --hybrid-overlay-cluster-subnets="
fi
fi
# start nbctl daemon for caching
export OVN_NB_DAEMON=$(ovn-nbctl --pidfile=/tmp/ovnk-nbctl.pid \
--detach \
-p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt \
--db "ssl:172.17.0.3:9641")
exec /usr/bin/ovnkube \
--init-master "${K8S_NODE}" \
--config-file=/run/ovnkube-config/ovnkube.conf \
--ovn-empty-lb-events \
--loglevel "${OVN_KUBE_LOG_LEVEL}" \
${hybrid_overlay_flags} \
--metrics-bind-address "0.0.0.0:9102" \
--sb-address "ssl://172.17.0.3:9642" \
--sb-client-privkey /ovn-cert/tls.key \
--sb-client-cert /ovn-cert/tls.crt \
--sb-client-cacert /ovn-ca/ca-bundle.crt
State: Waiting
Reason: CrashLoopBackOff
Last State: Terminated
Reason: Error
Message: 20:17:31.452738 1 shared_informer.go:227] caches populated
I0609 20:17:31.452741 1 shared_informer.go:227] caches populated
I0609 20:17:31.452745 1 shared_informer.go:227] caches populated
I0609 20:17:31.452749 1 shared_informer.go:227] caches populated
I0609 20:17:31.452840 1 ovnkube.go:315] Watching config file /run/ovnkube-config/ovnkube.conf for changes
I0609 20:17:31.452886 1 ovnkube.go:315] Watching config file /run/ovnkube-config/..2020_06_09_19_45_50.440900408/ovnkube.conf for changes
I0609 20:17:31.457368 1 leaderelection.go:242] attempting to acquire leader lease openshift-ovn-kubernetes/ovn-kubernetes-master...
I0609 20:17:31.464401 1 leaderelection.go:252] successfully acquired lease openshift-ovn-kubernetes/ovn-kubernetes-master
I0609 20:17:31.464562 1 master.go:57] won leader election; in active mode
E0609 20:17:31.487560 1 ovs.go:668] Failed to query OVN NB DB for SCTP support, stdout: "", stderr: "ovsdb-client: failed to connect to \"unix:/var/run/openvswitch/db.sock\" (No such file or directory)\n", error: OVN command '/usr/bin/ovsdb-client list-columns --data=bare --no-heading --format=json OVN_Northbound Load_Balancer' failed: exit status 1
E0609 20:17:31.487584 1 master.go:163] Failed to setup master (OVN command '/usr/bin/ovsdb-client list-columns --data=bare --no-heading --format=json OVN_Northbound Load_Balancer' failed: exit status 1)
panic: OVN command '/usr/bin/ovsdb-client list-columns --data=bare --no-heading --format=json OVN_Northbound Load_Balancer' failed: exit status 1
goroutine 159 [running]:
github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn.(*Controller).Start.func1(0x19cd5c0, 0xc0001ac2c0)
/home/brent/code/go/src/github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/master.go:65 +0x1d9
created by k8s.io/client-go/tools/leaderelection.(*LeaderElector).Run
/home/brent/code/go/src/github.com/ovn-org/ovn-kubernetes/go-controller/vendor/k8s.io/client-go/tools/leaderelection/leaderelection.go:207 +0x132
Exit Code: 2
Started: Tue, 09 Jun 2020 16:17:31 -0400
Finished: Tue, 09 Jun 2020 16:17:31 -0400
Ready: False
Restart Count: 11
Requests:
cpu: 100m
memory: 300Mi
Environment:
OVN_KUBE_LOG_LEVEL: 4
K8S_NODE: (v1:spec.nodeName)
Mounts:
/env from env-overrides (rw)
/etc/openvswitch/ from etc-openvswitch (rw)
/etc/ovn/ from etc-openvswitch (rw)
/ovn-ca from ovn-ca (rw)
/ovn-cert from ovn-cert (rw)
/run/openvswitch/ from run-openvswitch (rw)
/run/ovn/ from run-ovn (rw)
/run/ovnkube-config/ from ovnkube-config (rw)
/var/lib/openvswitch/ from var-lib-openvswitch (rw)
/var/run/secrets/kubernetes.io/serviceaccount from ovn-kubernetes-controller-token-5dqm4 (ro)
Conditions:
Type Status
Initialized True
Ready False
ContainersReady False
PodScheduled True
Volumes:
etc-openvswitch:
Type: HostPath (bare host directory volume)
Path: /var/lib/ovn/etc
HostPathType:
var-lib-openvswitch:
Type: HostPath (bare host directory volume)
Path: /var/lib/ovn/data
HostPathType:
run-openvswitch:
Type: EmptyDir (a temporary directory that shares a pod's lifetime)
Medium:
SizeLimit: <unset>
run-ovn:
Type: HostPath (bare host directory volume)
Path: /var/run/ovn
HostPathType:
ovnkube-config:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: ovnkube-config
Optional: false
env-overrides:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: env-overrides
Optional: true
ovn-ca:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: ovn-ca
Optional: false
ovn-cert:
Type: Secret (a volume populated by a Secret)
SecretName: ovn-cert
Optional: false
ovn-kubernetes-controller-token-5dqm4:
Type: Secret (a volume populated by a Secret)
SecretName: ovn-kubernetes-controller-token-5dqm4
Optional: false
QoS Class: Burstable
Node-Selectors: beta.kubernetes.io/os=linux
node-role.kubernetes.io/master=
Tolerations: node-role.kubernetes.io/master
node.kubernetes.io/disk-pressure:NoSchedule
node.kubernetes.io/memory-pressure:NoSchedule
node.kubernetes.io/network-unavailable
node.kubernetes.io/network-unavailable:NoSchedule
node.kubernetes.io/not-ready
node.kubernetes.io/not-ready:NoExecute
node.kubernetes.io/pid-pressure:NoSchedule
node.kubernetes.io/unreachable:NoExecute
node.kubernetes.io/unreachable
node.kubernetes.io/unschedulable:NoSchedule
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 32m default-scheduler Successfully assigned openshift-ovn-kubernetes/ovnkube-master-8khjb to ovn-control-plane
Normal Pulled 32m kubelet, ovn-control-plane Container image "origin-ovn-kubernetes:dev" already present on machine
Normal Created 32m kubelet, ovn-control-plane Created container northd
Normal Started 32m kubelet, ovn-control-plane Started container northd
Normal Pulled 32m kubelet, ovn-control-plane Container image "origin-ovn-kubernetes:dev" already present on machine
Normal Created 32m kubelet, ovn-control-plane Created container nbdb
Normal Started 32m kubelet, ovn-control-plane Started container nbdb
Normal Pulled 32m kubelet, ovn-control-plane Container image "origin-ovn-kubernetes:dev" already present on machine
Normal Created 32m kubelet, ovn-control-plane Created container sbdb
Normal Started 32m kubelet, ovn-control-plane Started container sbdb
Normal Pulled 31m (x4 over 32m) kubelet, ovn-control-plane Container image "origin-ovn-kubernetes:dev" already present on machine
Normal Created 31m (x4 over 32m) kubelet, ovn-control-plane Created container ovnkube-master
Normal Started 31m (x4 over 32m) kubelet, ovn-control-plane Started container ovnkube-master
Warning BackOff 2m44s (x141 over 32m) kubelet, ovn-control-plane Back-off restarting failed container
(='o'=)[brent@centos8-160:hack$]$
(='o'=)[brent@centos8-160:hack$]$
(='o'=)[brent@centos8-160:hack$]$ gpod
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
kube-system coredns-66bff467f8-9vnf5 0/1 Pending 0 40m <none> <none> <none> <none>
kube-system coredns-66bff467f8-wknqs 0/1 Pending 0 40m <none> <none> <none> <none>
kube-system etcd-ovn-control-plane 1/1 Running 0 40m 172.17.0.3 ovn-control-plane <none> <none>
kube-system kube-apiserver-ovn-control-plane 1/1 Running 0 40m 172.17.0.3 ovn-control-plane <none> <none>
kube-system kube-controller-manager-ovn-control-plane 1/1 Running 0 40m 172.17.0.3 ovn-control-plane <none> <none>
kube-system kube-proxy-kzll6 1/1 Running 0 40m 172.17.0.3 ovn-control-plane <none> <none>
kube-system kube-proxy-q5l5z 1/1 Running 0 39m 172.17.0.4 ovn-worker2 <none> <none>
kube-system kube-proxy-zbnq4 1/1 Running 0 39m 172.17.0.2 ovn-worker <none> <none>
kube-system kube-scheduler-ovn-control-plane 1/1 Running 0 40m 172.17.0.3 ovn-control-plane <none> <none>
local-path-storage local-path-provisioner-bd4bb6b75-ll9t6 0/1 Pending 0 40m <none> <none> <none> <none>
openshift-multus multus-42bkd 1/1 Running 2 33m 172.17.0.2 ovn-worker <none> <none>
openshift-multus multus-pgq9v 1/1 Running 2 33m 172.17.0.4 ovn-worker2 <none> <none>
openshift-multus multus-ztjjj 1/1 Running 2 33m 172.17.0.3 ovn-control-plane <none> <none>
openshift-network-operator network-operator-68f68bc9bc-rm64n 1/1 Running 0 35m 172.17.0.3 ovn-control-plane <none> <none>
openshift-ovn-kubernetes ovnkube-master-8khjb 3/4 CrashLoopBackOff 11 33m 172.17.0.3 ovn-control-plane <none> <none>
openshift-ovn-kubernetes ovnkube-node-h89h5 1/2 CrashLoopBackOff 6 33m 172.17.0.2 ovn-worker <none> <none>
openshift-ovn-kubernetes ovnkube-node-krhdv 1/2 CrashLoopBackOff 6 33m 172.17.0.3 ovn-control-plane <none> <none>
openshift-ovn-kubernetes ovnkube-node-xtblf 1/2 CrashLoopBackOff 6 33m 172.17.0.4 ovn-worker2 <none> <none>
openshift-ovn-kubernetes ovs-node-dgsbq 1/1 Running 0 33m 172.17.0.3 ovn-control-plane <none> <none>
openshift-ovn-kubernetes ovs-node-jgdrx 1/1 Running 0 33m 172.17.0.4 ovn-worker2 <none> <none>
openshift-ovn-kubernetes ovs-node-nvnks 1/1 Running 0 33m 172.17.0.2 ovn-worker <none> <none>
$kubectl describe pod ovnkube-node-h89h5 -n openshift-ovn-kubernetes
Name: ovnkube-node-h89h5
Namespace: openshift-ovn-kubernetes
Priority: 2000001000
Priority Class Name: system-node-critical
Node: ovn-worker/172.17.0.2
Start Time: Tue, 09 Jun 2020 15:45:50 -0400
Labels: app=ovnkube-node
component=network
controller-revision-hash=6fd8974898
kubernetes.io/os=linux
openshift.io/component=network
pod-template-generation=1
type=infra
Annotations: <none>
Status: Running
IP: 172.17.0.2
IPs:
IP: 172.17.0.2
Controlled By: DaemonSet/ovnkube-node
Containers:
ovn-controller:
Container ID: containerd://07738c1430081b27c6aeb318ab9a0d5bb4d5a4d659674199ec211d86fc083f86
Image: origin-ovn-kubernetes:dev
Image ID: sha256:05d261326bef5dfcf4e985361c941faf9292fb4ffbd920c72fddc86c4ded7603
Port: <none>
Host Port: <none>
Command:
/bin/bash
-c
set -e
if [[ -f "/env/${K8S_NODE}" ]]; then
set -o allexport
source "/env/${K8S_NODE}"
set +o allexport
fi
# Determine the ovn rundir.
if [[ -f /usr/bin/ovn-appctl ]] ; then
# ovn-appctl is present. Use new ovn run dir path.
OVNCTL_DIR=ovn
else
# ovn-appctl is not present. Use openvswitch run dir path.
OVNCTL_DIR=openvswitch
fi
exec ovn-controller unix:/var/run/openvswitch/db.sock -vfile:off \
--no-chdir --pidfile=/var/run/${OVNCTL_DIR}/ovn-controller.pid \
-p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt \
-vconsole:"${OVN_LOG_LEVEL}"
State: Running
Started: Tue, 09 Jun 2020 15:45:52 -0400
Ready: True
Restart Count: 0
Requests:
cpu: 100m
memory: 300Mi
Environment:
OVN_LOG_LEVEL: info
K8S_NODE: (v1:spec.nodeName)
Mounts:
/env from env-overrides (rw)
/etc/openvswitch from etc-openvswitch (rw)
/etc/ovn/ from etc-openvswitch (rw)
/ovn-ca from ovn-ca (rw)
/ovn-cert from ovn-cert (rw)
/run/openvswitch from run-openvswitch (rw)
/run/ovn/ from run-ovn (rw)
/var/lib/openvswitch from var-lib-openvswitch (rw)
/var/run/secrets/kubernetes.io/serviceaccount from ovn-kubernetes-node-token-x68k9 (ro)
ovnkube-node:
Container ID: containerd://b437e7ddd1c2ee18070eb08a303c5e7a6ec15ae5ecfef6b8262049efbca0ef42
Image: origin-ovn-kubernetes:dev
Image ID: sha256:05d261326bef5dfcf4e985361c941faf9292fb4ffbd920c72fddc86c4ded7603
Port: 9101/TCP
Host Port: 9101/TCP
Command:
/bin/bash
-c
set -xe
if [[ -f "/env/${K8S_NODE}" ]]; then
set -o allexport
source "/env/${K8S_NODE}"
set +o allexport
fi
cp -f /usr/libexec/cni/ovn-k8s-cni-overlay /cni-bin-dir/
ovn_config_namespace=openshift-ovn-kubernetes
retries=0
while true; do
db_ip=$(kubectl get ep -n ${ovn_config_namespace} ovnkube-db -o jsonpath='{.subsets[0].addresses[0].ip}')
if [[ -n "${db_ip}" ]]; then
break
fi
(( retries += 1 ))
if [[ "${retries}" -gt 40 ]]; then
echo "db endpoint never came up"
exit 1
fi
echo "waiting for db endpoint"
sleep 5
done
hybrid_overlay_flags=
if [[ -n "" ]]; then
hybrid_overlay_flags="--enable-hybrid-overlay"
if [[ -n "" ]]; then
hybrid_overlay_flags="${hybrid_overlay_flags} --hybrid-overlay-cluster-subnets="
fi
fi
exec /usr/bin/ovnkube --init-node "${K8S_NODE}" \
--nb-address "ssl://172.17.0.3:9641" \
--sb-address "ssl://172.17.0.3:9642" \
--nb-client-privkey /ovn-cert/tls.key \
--nb-client-cert /ovn-cert/tls.crt \
--nb-client-cacert /ovn-ca/ca-bundle.crt \
--sb-client-privkey /ovn-cert/tls.key \
--sb-client-cert /ovn-cert/tls.crt \
--sb-client-cacert /ovn-ca/ca-bundle.crt \
--config-file=/run/ovnkube-config/ovnkube.conf \
--loglevel "${OVN_KUBE_LOG_LEVEL}" \
${hybrid_overlay_flags} \
--metrics-bind-address "0.0.0.0:9101"
State: Waiting
Reason: CrashLoopBackOff
Last State: Terminated
Reason: Error
Message: '
waiting for db endpoint
+ db_ip=
+ [[ -n '' ]]
+ (( retries += 1 ))
+ [[ 33 -gt 40 ]]
+ echo 'waiting for db endpoint'
+ sleep 5
+ true
++ kubectl get ep -n openshift-ovn-kubernetes ovnkube-db -o 'jsonpath={.subsets[0].addresses[0].ip}'
+ db_ip=
+ [[ -n '' ]]
waiting for db endpoint
+ (( retries += 1 ))
+ [[ 34 -gt 40 ]]
+ echo 'waiting for db endpoint'
+ sleep 5
+ true
++ kubectl get ep -n openshift-ovn-kubernetes ovnkube-db -o 'jsonpath={.subsets[0].addresses[0].ip}'
waiting for db endpoint
+ db_ip=
+ [[ -n '' ]]
+ (( retries += 1 ))
+ [[ 35 -gt 40 ]]
+ echo 'waiting for db endpoint'
+ sleep 5
+ true
++ kubectl get ep -n openshift-ovn-kubernetes ovnkube-db -o 'jsonpath={.subsets[0].addresses[0].ip}'
+ db_ip=
+ [[ -n '' ]]
+ (( retries += 1 ))
+ [[ 36 -gt 40 ]]
+ echo 'waiting for db endpoint'
+ sleep 5
waiting for db endpoint
+ true
++ kubectl get ep -n openshift-ovn-kubernetes ovnkube-db -o 'jsonpath={.subsets[0].addresses[0].ip}'
+ db_ip=
+ [[ -n '' ]]
+ (( retries += 1 ))
+ [[ 37 -gt 40 ]]
+ echo 'waiting for db endpoint'
+ sleep 5
waiting for db endpoint
+ true
++ kubectl get ep -n openshift-ovn-kubernetes ovnkube-db -o 'jsonpath={.subsets[0].addresses[0].ip}'
+ db_ip=
+ [[ -n '' ]]
+ (( retries += 1 ))
+ [[ 38 -gt 40 ]]
+ echo 'waiting for db endpoint'
+ sleep 5
waiting for db endpoint
+ true
++ kubectl get ep -n openshift-ovn-kubernetes ovnkube-db -o 'jsonpath={.subsets[0].addresses[0].ip}'
waiting for db endpoint
+ db_ip=
+ [[ -n '' ]]
+ (( retries += 1 ))
+ [[ 39 -gt 40 ]]
+ echo 'waiting for db endpoint'
+ sleep 5
+ true
++ kubectl get ep -n openshift-ovn-kubernetes ovnkube-db -o 'jsonpath={.subsets[0].addresses[0].ip}'
+ db_ip=
+ [[ -n '' ]]
+ (( retries += 1 ))
+ [[ 40 -gt 40 ]]
+ echo 'waiting for db endpoint'
+ sleep 5
waiting for db endpoint
+ true
++ kubectl get ep -n openshift-ovn-kubernetes ovnkube-db -o 'jsonpath={.subsets[0].addresses[0].ip}'
+ db_ip=
+ [[ -n '' ]]
+ (( retries += 1 ))
+ [[ 41 -gt 40 ]]
+ echo 'db endpoint never came up'
+ exit 1
db endpoint never came up
Exit Code: 1
Started: Tue, 09 Jun 2020 16:12:42 -0400
Finished: Tue, 09 Jun 2020 16:16:09 -0400
Ready: False
Restart Count: 6
Requests:
cpu: 100m
memory: 300Mi
Readiness: exec [test -f /etc/cni/net.d/10-ovn-kubernetes.conf] delay=5s timeout=1s period=5s #success=1 #failure=3
Environment:
KUBERNETES_SERVICE_PORT: 6443
KUBERNETES_SERVICE_HOST: 172.17.0.3
OVN_KUBE_LOG_LEVEL: 4
K8S_NODE: (v1:spec.nodeName)
Mounts:
/cni-bin-dir from host-cni-bin (rw)
/env from env-overrides (rw)
/etc/cni/net.d from host-cni-netd (rw)
/etc/openvswitch from etc-openvswitch (rw)
/etc/ovn/ from etc-openvswitch (rw)
/host from host-slash (ro)
/ovn-ca from ovn-ca (rw)
/ovn-cert from ovn-cert (rw)
/run/netns from host-run-netns (rw)
/run/openvswitch from run-openvswitch (rw)
/run/ovn-kubernetes/ from host-run-ovn-kubernetes (rw)
/run/ovn/ from run-ovn (rw)
/run/ovnkube-config/ from ovnkube-config (rw)
/var/lib/cni/networks/ovn-k8s-cni-overlay from host-var-lib-cni-networks-ovn-kubernetes (rw)
/var/lib/openvswitch from var-lib-openvswitch (rw)
/var/run/secrets/kubernetes.io/serviceaccount from ovn-kubernetes-node-token-x68k9 (ro)
Conditions:
Type Status
Initialized True
Ready False
ContainersReady False
PodScheduled True
Volumes:
host-slash:
Type: HostPath (bare host directory volume)
Path: /
HostPathType:
host-run-netns:
Type: HostPath (bare host directory volume)
Path: /run/netns
HostPathType:
var-lib-openvswitch:
Type: HostPath (bare host directory volume)
Path: /var/lib/openvswitch/data
HostPathType:
etc-openvswitch:
Type: HostPath (bare host directory volume)
Path: /var/lib/openvswitch/etc
HostPathType:
run-openvswitch:
Type: HostPath (bare host directory volume)
Path: /var/run/openvswitch
HostPathType:
run-ovn:
Type: HostPath (bare host directory volume)
Path: /var/run/ovn
HostPathType:
host-run-ovn-kubernetes:
Type: HostPath (bare host directory volume)
Path: /run/ovn-kubernetes
HostPathType:
host-cni-bin:
Type: HostPath (bare host directory volume)
Path: /var/lib/cni/bin
HostPathType:
host-cni-netd:
Type: HostPath (bare host directory volume)
Path: /var/run/multus/cni/net.d
HostPathType:
host-var-lib-cni-networks-ovn-kubernetes:
Type: HostPath (bare host directory volume)
Path: /var/lib/cni/networks/ovn-k8s-cni-overlay
HostPathType:
ovnkube-config:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: ovnkube-config
Optional: false
env-overrides:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: env-overrides
Optional: true
ovn-ca:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: ovn-ca
Optional: false
ovn-cert:
Type: Secret (a volume populated by a Secret)
SecretName: ovn-cert
Optional: false
ovn-kubernetes-node-token-x68k9:
Type: Secret (a volume populated by a Secret)
SecretName: ovn-kubernetes-node-token-x68k9
Optional: false
QoS Class: Burstable
Node-Selectors: beta.kubernetes.io/os=linux
Tolerations:
node.kubernetes.io/disk-pressure:NoSchedule
node.kubernetes.io/memory-pressure:NoSchedule
node.kubernetes.io/network-unavailable:NoSchedule
node.kubernetes.io/not-ready:NoExecute
node.kubernetes.io/pid-pressure:NoSchedule
node.kubernetes.io/unreachable:NoExecute
node.kubernetes.io/unschedulable:NoSchedule
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 33m default-scheduler Successfully assigned openshift-ovn-kubernetes/ovnkube-node-h89h5 to ovn-worker
Normal Pulled 33m kubelet, ovn-worker Container image "origin-ovn-kubernetes:dev" already present on machine
Normal Created 33m kubelet, ovn-worker Created container ovn-controller
Normal Started 33m kubelet, ovn-worker Started container ovn-controller
Normal Pulled 33m kubelet, ovn-worker Container image "origin-ovn-kubernetes:dev" already present on machine
Normal Created 33m kubelet, ovn-worker Created container ovnkube-node
Normal Started 33m kubelet, ovn-worker Started container ovnkube-node
Warning BackOff 8m28s (x21 over 26m) kubelet, ovn-worker Back-off restarting failed container
Warning Unhealthy 3m33s (x282 over 33m) kubelet, ovn-worker Readiness probe failed:
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment