Skip to content

Instantly share code, notes, and snippets.

@dharmit

dharmit/3.10 Secret

Last active November 19, 2018 08:31
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save dharmit/e1e12096c056825e69094be21e7378f2 to your computer and use it in GitHub Desktop.
Save dharmit/e1e12096c056825e69094be21e7378f2 to your computer and use it in GitHub Desktop.
Download ZIP
Upgrade from 3.9 to 3.10 fails while waiting for master node to be ready
Raw
3.10
# Create an OSEv3 group that contains the masters and nodes groups
[OSEv3:children]
masters
nodes
etcd
# Set variables common for all OSEv3 hosts
[OSEv3:vars]
# SSH user, this user should allow ssh based auth without requiring a password
ansible_ssh_user=root
openshift_node_groups=[{'name': 'ccp-openshift-master', 'labels': ['node-role.kubernetes.io/master=true', 'node-role.kubernetes.io/node-type=metrics', 'node-role.kubernetes.io/zone=default', 'node-role.kubernetes.io/infra=true'], 'edits': [{'key': 'kubeletArguments.max-pods', 'value': ['80']}, {'key': 'kubeletArguments.image-gc-high-threshold', 'value': ['70']}, {'key': 'kubeletArguments.minimum-container-ttl-duration', 'value': '2h'}, {'key': 'kubeletArguments.maximum-dead-containers', 'value': '50'}, {'key': 'kubeletArguments.maximum-dead-containers-per-container', 'value' : '2'}]}, {'name': 'ccp-openshift-node', 'labels': ['node-role.kubernetes.io/node-type=logging', 'node-role.kubernetes.io/zone=default'], 'edits': [{'key': 'kubeletArguments.max-pods', 'value': ['80']}, {'key': 'kubeletArguments.image-gc-high-threshold', 'value': ['70']}, {'key': 'kubeletArguments.minimum-container-ttl-duration', 'value': '2h'}, {'key': 'kubeletArguments.maximum-dead-containers', 'value': '50'}, {'key': 'kubeletArguments.maximum-dead-containers-per-container', 'value' : '2'}]}]
# If ansible_ssh_user is not root, ansible_become must be set to true
# ansible_become=true
# containerized=true
debug_level=4
openshift_master_api_port=8443
openshift_deployment_type=origin
openshift_release=v3.10
os_firewall_use_firewalld=false
openshift_disable_swap=false
openshift_clock_enabled=false
openshift_pkg_version=-3.10.0
openshift_enable_service_catalog=false
openshift_docker_insecure_registries=192.168.122.25:5000
openshift_docker_additional_registries=192.168.122.25:5000, registry.centos.org
openshift_master_default_subdomain={{ hostvars[groups['masters'][0]].openshift_ip }}.nip.io
openshift_node_kubelet_args={'max-pods': ['80'], 'image-gc-high-threshold': ['80'], 'image-gc-low-threshold': ['70'], 'minimum-container-ttl-duration': '2h', 'maximum-dead-containers': '50', 'maximum-dead-containers-per-container': '2'}
openshift_rolling_restart_mode=system
# prometheus stack
openshift_hosted_prometheus_deploy=true
openshift_prometheus_node_selector={'node-role.kubernetes.io/node-type': 'metrics'}
openshift_prometheus_storage_type=pvc
openshift_prometheus_alertmanager_storage_type=pvc
openshift_prometheus_alertbuffer_storage_type=pvc
# openshift_prometheus_additional_rules_file=/root/openshift-cluster/rules.example
# grafana deployment
openshift_hosted_grafana_deploy=true
# uncomment the following to enable htpasswd authentication; defaults to DenyAllPasswordIdentityProvider
openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider', 'filename': '/etc/origin/master/htpasswd'}]
openshift_master_htpasswd_users={'dev': '$apr1$aIR9qk/K$kdEBbyak6ciQcwLnABi.b/'}
# default selectors for router and registry services
openshift_hosted_router_selector='node-role.kubernetes.io/infra=true'
openshift_registry_selector='node-role.kubernetes.io/infra=true'
openshift_disable_check=docker_storage,memory_availability
# host group for masters
[masters]
os-master-1.example.com
# host group for etcd
[etcd]
os-master-1.example.com
# host group for nodes, includes region info
[nodes]
os-master-1.example.com openshift_node_group_name='ccp-openshift-master' openshift_schedulable=true openshift_ip=192.168.122.167
os-node-1.example.com openshift_node_group_name='ccp-openshift-node' openshift_schedulable=true openshift_ip=192.168.122.216
os-node-2.example.com openshift_node_group_name='ccp-openshift-node' openshift_schedulable=true openshift_ip=192.168.122.52
os-node-3.example.com openshift_node_group_name='ccp-openshift-node' openshift_schedulable=true openshift_ip=192.168.122.150
os-node-4.example.com openshift_node_group_name='ccp-openshift-node' openshift_schedulable=true openshift_ip=192.168.122.43
os-node-5.example.com openshift_node_group_name='ccp-openshift-node' openshift_schedulable=true openshift_ip=192.168.122.65
os-node-6.example.com openshift_node_group_name='ccp-openshift-node' openshift_schedulable=true openshift_ip=192.168.122.109
os-node-7.example.com openshift_node_group_name='ccp-openshift-node' openshift_schedulable=true openshift_ip=192.168.122.206
os-node-8.example.com openshift_node_group_name='ccp-openshift-node' openshift_schedulable=true openshift_ip=192.168.122.236
os-node-9.example.com openshift_node_group_name='ccp-openshift-node' openshift_schedulable=true openshift_ip=192.168.122.78
os-node-10.example.com openshift_node_group_name='ccp-openshift-node' openshift_schedulable=true openshift_ip=192.168.122.181
Raw
3.9
# Create an OSEv3 group that contains the masters and nodes groups
[OSEv3:children]
masters
nodes
etcd
# Set variables common for all OSEv3 hosts
[OSEv3:vars]
# SSH user, this user should allow ssh based auth without requiring a password
ansible_ssh_user=root
# If ansible_ssh_user is not root, ansible_become must be set to true
# ansible_become=true
# containerized=true
debug_level=4
openshift_master_api_port=8443
openshift_deployment_type=origin
openshift_release=v3.9
os_firewall_use_firewalld=false
openshift_disable_swap=false
openshift_clock_enabled=false
openshift_pkg_version=-3.9.0
openshift_enable_service_catalog=false
openshift_docker_insecure_registries=192.168.122.25:5000
openshift_docker_additional_registries=192.168.122.25:5000, registry.centos.org
openshift_master_default_subdomain={{ hostvars[groups['masters'][0]].openshift_ip }}.nip.io
# openshift_rolling_restart_mode=system
# prometheus stack
openshift_hosted_prometheus_deploy=true
openshift_prometheus_node_selector={'size': 'large3'}
openshift_prometheus_storage_type=pvc
openshift_prometheus_alertmanager_storage_type=pvc
openshift_prometheus_alertbuffer_storage_type=pvc
# openshift_prometheus_additional_rules_file=/root/openshift-cluster/rules.example
# grafana deployment
openshift_hosted_grafana_deploy=true
# uncomment the following to enable htpasswd authentication; defaults to DenyAllPasswordIdentityProvider
openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider', 'filename': '/etc/origin/master/htpasswd'}]
openshift_master_htpasswd_users={'dev': '$apr1$aIR9qk/K$kdEBbyak6ciQcwLnABi.b/'}
# default selectors for router and registry services
openshift_router_selector='region=infra'
openshift_registry_selector='region=infra'
openshift_disable_check=docker_storage,memory_availability
# host group for masters
[masters]
os-master-1.example.com
# host group for etcd
[etcd]
os-master-1.example.com
# host group for nodes, includes region info
[nodes]
os-master-1.example.com openshift_node_labels="{'region': 'infra','zone': 'default','purpose':'infra', 'node-type': 'metrics', 'size': 'large1'}" openshift_schedulable=true openshift_ip=192.168.122.167
os-node-1.example.com openshift_node_labels="{'region':'primary','zone': 'default','purpose':'prod', 'node-type': 'logging', 'size': 'large2'}" openshift_schedulable=true openshift_ip=192.168.122.216
os-node-2.example.com openshift_node_labels="{'region':'primary','zone': 'default','purpose':'prod', 'node-type': 'logging'}" openshift_schedulable=true openshift_ip=192.168.122.52
os-node-3.example.com openshift_node_labels="{'region':'primary','zone': 'default','purpose':'prod', 'node-type': 'logging'}" openshift_schedulable=true openshift_ip=192.168.122.150
os-node-4.example.com openshift_node_labels="{'region':'primary','zone': 'default','purpose':'prod', 'node-type': 'logging', 'size': 'large3'}" openshift_schedulable=true openshift_ip=192.168.122.43
os-node-5.example.com openshift_node_labels="{'region':'primary','zone': 'default','purpose':'prod', 'node-type': 'logging'}" openshift_schedulable=true openshift_ip=192.168.122.65
os-node-6.example.com openshift_node_labels="{'region':'primary','zone': 'default','purpose':'prod', 'node-type': 'logging'}" openshift_schedulable=true openshift_ip=192.168.122.109
os-node-7.example.com openshift_node_labels="{'region':'primary','zone': 'default','purpose':'prod', 'node-type': 'logging'}" openshift_schedulable=true openshift_ip=192.168.122.206
os-node-8.example.com openshift_node_labels="{'region':'primary','zone': 'default','purpose':'prod', 'node-type': 'logging'}" openshift_schedulable=true openshift_ip=192.168.122.236
os-node-9.example.com openshift_node_labels="{'region':'primary','zone': 'default','purpose':'prod', 'node-type': 'logging'}" openshift_schedulable=true openshift_ip=192.168.122.78
os-node-10.example.com openshift_node_labels="{'region':'primary','zone': 'default','purpose':'prod', 'node-type': 'logging'}" openshift_schedulable=true openshift_ip=192.168.122.181
Raw
journalctl_xe
-- Logs begin at Mon 2018-11-19 06:04:45 UTC, end at Mon 2018-11-19 08:28:35 UTC. --
Nov 19 08:28:24 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:24.521040 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.02148ms)
Nov 19 08:28:24 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:24.619109 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:24.619078243 +0000 UTC m=+3282.253492445)
Nov 19 08:28:24 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:24.620723 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:24 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:24.620831 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.741627ms)
Nov 19 08:28:24 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:24.718234 40659 wrap.go:42] GET /api/v1/namespaces/kube-system: (4.6766ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 127.0.0.1:60614]
Nov 19 08:28:24 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:24.719201 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:24.719173663 +0000 UTC m=+3282.353587888)
Nov 19 08:28:24 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:24.721487 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:24 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:24.721598 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.412596ms)
Nov 19 08:28:24 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:24.722965 40659 wrap.go:42] GET /api/v1/namespaces/kube-public: (2.95594ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 127.0.0.1:60614]
Nov 19 08:28:24 os-master-1.example.com origin-node[62740]: I1119 08:28:24.818301 62740 kubelet.go:1961] SyncLoop (housekeeping)
Nov 19 08:28:24 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:24.819096 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:24.819069032 +0000 UTC m=+3282.453483236)
Nov 19 08:28:24 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:24.820691 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:24 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:24.820789 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.709059ms)
Nov 19 08:28:24 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:24.919127 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:24.91910681 +0000 UTC m=+3282.553521011)
Nov 19 08:28:24 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:24.921961 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:24 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:24.922032 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.916204ms)
Nov 19 08:28:24 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:24.997997 40659 wrap.go:42] GET /api/v1/namespaces/openshift-node/secrets/sync-token-f67g7?resourceVersion=0: (5.38253ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.42:60392]
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.018979 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:25.018959102 +0000 UTC m=+3282.653373303)
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.020483 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.020593 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.623336ms)
Nov 19 08:28:25 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:25.043386 40659 wrap.go:42] GET /api/v1/nodes/os-node-3.example.com?resourceVersion=0&timeout=10s: (1.65585ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.136:50920]
Nov 19 08:28:25 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:25.063995 40659 wrap.go:42] PATCH /api/v1/nodes/os-node-3.example.com/status?timeout=10s: (6.772028ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.136:50920]
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.065939 40820 attach_detach_controller.go:493] processVolumesInUse for node "os-node-3.example.com"
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.119065 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:25.119030998 +0000 UTC m=+3282.753445203)
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.120826 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.120906 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.865752ms)
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.218998 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:25.218965274 +0000 UTC m=+3282.853379500)
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.221537 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.221601 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.627196ms)
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.318997 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:25.31896653 +0000 UTC m=+3282.953380793)
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.321517 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.321604 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.62353ms)
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.418992 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:25.418968188 +0000 UTC m=+3283.053382406)
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.420953 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.421055 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.075671ms)
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.518946 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:25.518919336 +0000 UTC m=+3283.153333529)
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.520494 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.520597 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.667173ms)
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.618924 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:25.618869629 +0000 UTC m=+3283.253283851)
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.620672 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.620745 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.853994ms)
Nov 19 08:28:25 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:25.519669 40659 wrap.go:42] GET /api/v1/namespaces/openshift-node/secrets/sync-dockercfg-g6b6k?resourceVersion=0: (251.120265ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.42:60392]
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: W1119 08:28:25.470854 62740 cni.go:171] Unable to update cni config: No networks found in /etc/cni/net.d
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.471393 62740 kubelet.go:2140] Container runtime status: Runtime Conditions: RuntimeReady=true reason: message:, NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: E1119 08:28:25.471439 62740 kubelet.go:2143] Container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.718973 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:25.718941445 +0000 UTC m=+3283.353355655)
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.720825 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.720921 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.969686ms)
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.818204 62740 kubelet.go:1938] SyncLoop (SYNC): 1 pods; sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.818943 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:25.818914238 +0000 UTC m=+3283.453328469)
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.818462 62740 kubelet_pods.go:1372] Generating status for "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.818855 62740 status_manager.go:340] Ignoring same status for pod "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)", status: {Phase:Running Conditions:[{Type:Initialized Status:True LastProbeTime:0001-01-01 00:00:00 +0000 UTC LastTransitionTime:2018-11-19 07:53:51 +0000 UTC Reason: Message:} {Type:Ready Status:False LastProbeTime:0001-01-01 00:00:00 +0000 UTC LastTransitionTime:2018-11-19 08:23:25 +0000 UTC Reason:ContainersNotReady Message:containers with unready status: [sdn]} {Type:PodScheduled Status:True LastProbeTime:0001-01-01 00:00:00 +0000 UTC LastTransitionTime:2018-11-19 07:54:12 +0000 UTC Reason: Message:}] Message: Reason: NominatedNodeName: HostIP:192.168.122.91 PodIP:192.168.122.91 StartTime:2018-11-19 07:53:51 +0000 UTC InitContainerStatuses:[] ContainerStatuses:[{Name:sdn State:{Waiting:&ContainerStateWaiting{Reason:CrashLoopBackOff,Message:Back-off 5m0s restarting failed container=sdn pod=sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9),} Running:nil Terminated:nil} LastTerminationState:{Waiting:nil Running:nil Terminated:&ContainerStateTerminated{ExitCode:255,Signal:0,Reason:Error,Message:,StartedAt:2018-11-19 08:23:23 +0000 UTC,FinishedAt:2018-11-19 08:23:25 +0000 UTC,ContainerID:docker://d9b4320360a032613339922e004acf1b470607b6ee6f56249f2236d365e31264,}} Ready:false RestartCount:10 Image:docker.io/openshift/node:v3.10.0 ImageID:docker-pullable://docker.io/openshift/node@sha256:a6294e3d1bd6459c20e231b3276fb9dd47a8ac2db8c6a3cd258c7499e0d1d2a3 ContainerID:docker://d9b4320360a032613339922e004acf1b470607b6ee6f56249f2236d365e31264}] QOSClass:Burstable}
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.819642 62740 volume_manager.go:347] Waiting for volumes to attach and mount for pod "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.820572 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.820687 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.763829ms)
Nov 19 08:28:25 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:25.825585 40659 wrap.go:42] GET /api/v1/nodes/os-master-1.example.com?resourceVersion=0&timeout=10s: (1.173637ms) 200 [[hyperkube/v1.10.0+b81c8f8 (linux/amd64) kubernetes/b81c8f8] 192.168.122.91:35472]
Nov 19 08:28:25 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:25.845430 40659 wrap.go:42] PATCH /api/v1/nodes/os-master-1.example.com/status?timeout=10s: (7.661118ms) 200 [[hyperkube/v1.10.0+b81c8f8 (linux/amd64) kubernetes/b81c8f8] 192.168.122.91:35472]
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.847115 40820 attach_detach_controller.go:493] processVolumesInUse for node "os-master-1.example.com"
Nov 19 08:28:25 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:25.905848 40659 wrap.go:42] GET /api/v1/namespaces/openshift-node/secrets/sync-token-f67g7: (4.554641ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:25 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:25.909672 40659 wrap.go:42] GET /api/v1/namespaces/openshift-node/serviceaccounts/sync: (2.679335ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.909836 62740 desired_state_of_world_populator.go:302] Added volume "host-config" (volSpec="host-config") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.909947 62740 desired_state_of_world_populator.go:302] Added volume "host-sysconfig-node" (volSpec="host-sysconfig-node") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.910004 62740 desired_state_of_world_populator.go:302] Added volume "host-modules" (volSpec="host-modules") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.910050 62740 desired_state_of_world_populator.go:302] Added volume "host-var-run" (volSpec="host-var-run") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.910096 62740 desired_state_of_world_populator.go:302] Added volume "host-var-run-dbus" (volSpec="host-var-run-dbus") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.910142 62740 desired_state_of_world_populator.go:302] Added volume "host-var-run-ovs" (volSpec="host-var-run-ovs") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.910204 62740 desired_state_of_world_populator.go:302] Added volume "host-var-run-kubernetes" (volSpec="host-var-run-kubernetes") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.910270 62740 desired_state_of_world_populator.go:302] Added volume "host-var-run-openshift-sdn" (volSpec="host-var-run-openshift-sdn") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.910354 62740 desired_state_of_world_populator.go:302] Added volume "host-opt-cni-bin" (volSpec="host-opt-cni-bin") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.910402 62740 desired_state_of_world_populator.go:302] Added volume "host-etc-cni-netd" (volSpec="host-etc-cni-netd") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.910447 62740 desired_state_of_world_populator.go:302] Added volume "host-var-lib-cni-networks-openshift-sdn" (volSpec="host-var-lib-cni-networks-openshift-sdn") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:25 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:25.914379 40659 wrap.go:42] GET /api/v1/namespaces/openshift-node/configmaps/ccp-openshift-master?timeout=10s: (14.707763ms) 200 [[oc/v1.10.0+b81c8f8 (linux/amd64) kubernetes/b81c8f8] 192.168.122.91:36466]
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.918945 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:25.918929385 +0000 UTC m=+3283.553343600)
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.920584 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:25 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:25.920680 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.744757ms)
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.910501 62740 desired_state_of_world_populator.go:302] Added volume "sdn-token-5rn77" (volSpec="sdn-token-5rn77") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.925141 62740 reconciler.go:237] Starting operationExecutor.MountVolume for volume "sdn-token-5rn77" (UniqueName: "kubernetes.io/secret/40bf4573-ebd0-11e8-8b2e-5254005db2c9-sdn-token-5rn77") pod "sdn-pgmqr" (UID: "40bf4573-ebd0-11e8-8b2e-5254005db2c9") Volume is already mounted to pod, but remount was requested.
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.925520 62740 secret.go:186] Setting up volume sdn-token-5rn77 for pod 40bf4573-ebd0-11e8-8b2e-5254005db2c9 at /var/lib/origin/openshift.local.volumes/pods/40bf4573-ebd0-11e8-8b2e-5254005db2c9/volumes/kubernetes.io~secret/sdn-token-5rn77
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.925758 62740 secret.go:217] Received secret openshift-sdn/sdn-token-5rn77 containing (4) pieces of data, 4115 total bytes
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.926369 62740 atomic_writer.go:156] pod openshift-sdn/sdn-pgmqr volume sdn-token-5rn77: no update required for target directory /var/lib/origin/openshift.local.volumes/pods/40bf4573-ebd0-11e8-8b2e-5254005db2c9/volumes/kubernetes.io~secret/sdn-token-5rn77
Nov 19 08:28:25 os-master-1.example.com origin-node[62740]: I1119 08:28:25.926420 62740 operation_generator.go:555] MountVolume.SetUp succeeded for volume "sdn-token-5rn77" (UniqueName: "kubernetes.io/secret/40bf4573-ebd0-11e8-8b2e-5254005db2c9-sdn-token-5rn77") pod "sdn-pgmqr" (UID: "40bf4573-ebd0-11e8-8b2e-5254005db2c9")
Nov 19 08:28:25 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:25.984331 40659 wrap.go:42] GET /api/v1/namespaces/kube-system/configmaps/kube-scheduler: (43.073249ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/leader-election] 192.168.122.91:60300]
Nov 19 08:28:26 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:25.999738 40659 wrap.go:42] PUT /api/v1/namespaces/kube-system/configmaps/kube-scheduler: (8.609721ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/leader-election] 192.168.122.91:60300]
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.000387 40820 leaderelection.go:199] successfully renewed lease kube-system/kube-scheduler
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.018984 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:26.018964833 +0000 UTC m=+3283.653379026)
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.020766 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.020874 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.899679ms)
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.119058 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:26.119025636 +0000 UTC m=+3283.753439855)
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.121024 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.121119 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.084052ms)
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: I1119 08:28:26.121299 62740 volume_manager.go:380] All volumes are attached and mounted for pod "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: I1119 08:28:26.121991 62740 kuberuntime_manager.go:513] Container {Name:sdn Image:docker.io/openshift/origin-node:v3.10 Command:[/bin/bash -c #!/bin/bash
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: set -euo pipefail
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: # if another process is listening on the cni-server socket, wait until it exits
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: trap 'kill $(jobs -p); exit 0' TERM
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: retries=0
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: while true; do
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: if echo 'test' | socat - UNIX-CONNECT:/var/run/openshift-sdn/cni-server.sock >/dev/null; then
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: echo "warning: Another process is currently listening on the CNI socket, waiting 15s ..." 2>&1
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: sleep 15 & wait
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: (( retries += 1 ))
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: else
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: break
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: if [[ "${retries}" -gt 40 ]]; then
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: echo "error: Another process is currently listening on the CNI socket, exiting" 2>&1
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: exit 1
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: done
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: # if the node config doesn't exist yet, wait until it does
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: retries=0
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: while true; do
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: if [[ ! -f /etc/origin/node/node-config.yaml ]]; then
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: echo "warning: Cannot find existing node-config.yaml, waiting 15s ..." 2>&1
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: sleep 15 & wait
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: (( retries += 1 ))
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: else
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: break
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: if [[ "${retries}" -gt 40 ]]; then
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: echo "error: No existing node-config.yaml, exiting" 2>&1
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: exit 1
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: done
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: # Take over network functions on the node
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: rm -Rf /etc/cni/net.d/*
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: rm -Rf /host/opt/cni/bin/*
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: cp -Rf /opt/cni/bin/* /host/opt/cni/bin/
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: if [[ -f /etc/sysconfig/origin-node ]]; then
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: set -o allexport
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: source /etc/sysconfig/origin-node
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: # use either the bootstrapped node kubeconfig or the static configuration
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: file=/etc/origin/node/node.kubeconfig
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: if [[ ! -f "${file}" ]]; then
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: # use the static node config if it exists
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: # TODO: remove when static node configuration is no longer supported
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: for f in /etc/origin/node/system*.kubeconfig; do
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: echo "info: Using ${f} for node configuration" 1>&2
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: file="${f}"
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: break
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: done
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: # Use the same config as the node, but with the service account token
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: oc config "--config=${file}" view --flatten > /tmp/kubeconfig
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: oc config --config=/tmp/kubeconfig set-credentials sa "--token=$( cat /var/run/secrets/kubernetes.io/serviceaccount/token )"
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: oc config --config=/tmp/kubeconfig set-context "$( oc config --config=/tmp/kubeconfig current-context )" --user=sa
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: # Launch the network process
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: exec openshift start network --config=/etc/origin/node/node-config.yaml --kubeconfig=/tmp/kubeconfig --loglevel=${DEBUG_LOGLEVEL:-2}
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: ] Args:[] WorkingDir: Ports:[{Name:healthz HostPort:10256 ContainerPort:10256 Protocol:TCP HostIP:}] EnvFrom:[] Env:[{Name:OPENSHIFT_DNS_DOMAIN Value:cluster.local ValueFrom:nil}] Resources:{Limits:map[] Requests:map[cpu:{i:{value:100 scale:-3} d:{Dec:<nil>} s:100m Format:DecimalSI} memory:{i:{value:209715200 scale:0} d:{Dec:<nil>} s: Format:BinarySI}]} VolumeMounts:[{Name:host-config ReadOnly:true MountPath:/etc/origin/node/ SubPath: MountPropagation:<nil>} {Name:host-sysconfig-node ReadOnly:true MountPath:/etc/sysconfig/origin-node SubPath: MountPropagation:<nil>} {Name:host-var-run ReadOnly:false MountPath:/var/run SubPath: MountPropagation:<nil>} {Name:host-var-run-dbus ReadOnly:true MountPath:/var/run/dbus/ SubPath: MountPropagation:<nil>} {Name:host-var-run-ovs ReadOnly:true MountPath:/var/run/openvswitch/ SubPath: MountPropagation:<nil>} {Name:host-var-run-kubernetes ReadOnly:true MountPath:/var/run/kubernetes/ SubPath: MountPropagation:<nil>} {Name:host-var-run-openshift-sdn ReadOnly:false MountPath:/var/run/openshift-sdn SubPath: MountPropagation:<nil>} {Name:host-opt-cni-bin ReadOnly:false MountPath:/host/opt/cni/bin SubPath: MountPropagation:<nil>} {Name:host-etc-cni-netd ReadOnly:false MountPath:/etc/cni/net.d SubPath: MountPropagation:<nil>} {Name:host-var-lib-cni-networks-openshift-sdn ReadOnly:false MountPath:/var/lib/cni/networks/openshift-sdn SubPath: MountPropagation:<nil>} {Name:sdn-token-5rn77 ReadOnly:true MountPath:/var/run/secrets/kubernetes.io/serviceaccount SubPath: MountPropagation:<nil>}] VolumeDevices:[] LivenessProbe:nil ReadinessProbe:nil Lifecycle:nil TerminationMessagePath:/dev/termination-log TerminationMessagePolicy:File ImagePullPolicy:IfNotPresent SecurityContext:&SecurityContext{Capabilities:nil,Privileged:*true,SELinuxOptions:nil,RunAsUser:*0,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,} Stdin:false StdinOnce:false TTY:false} is dead, but RestartPolicy says that we should restart it.
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: I1119 08:28:26.122053 62740 kuberuntime_manager.go:570] computePodActions got {KillPod:false CreateSandbox:false SandboxID:5fd7fcbd62c89c72ef83f0b96462ec355dd845d0e1a221eb349a78408fac2208 Attempt:0 NextInitContainerToStart:nil ContainersToStart:[0] ContainersToKill:map[]} for pod "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: I1119 08:28:26.122372 62740 kuberuntime_manager.go:757] checking backoff for container "sdn" in pod "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: I1119 08:28:26.122622 62740 kuberuntime_manager.go:724] Creating container &Container{Name:sdn,Image:docker.io/openshift/origin-node:v3.10,Command:[/bin/bash -c #!/bin/bash
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: set -euo pipefail
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: # if another process is listening on the cni-server socket, wait until it exits
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: trap 'kill $(jobs -p); exit 0' TERM
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: retries=0
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: while true; do
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: if echo 'test' | socat - UNIX-CONNECT:/var/run/openshift-sdn/cni-server.sock >/dev/null; then
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: echo "warning: Another process is currently listening on the CNI socket, waiting 15s ..." 2>&1
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: sleep 15 & wait
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: (( retries += 1 ))
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: else
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: break
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: if [[ "${retries}" -gt 40 ]]; then
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: echo "error: Another process is currently listening on the CNI socket, exiting" 2>&1
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: exit 1
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: done
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: # if the node config doesn't exist yet, wait until it does
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: retries=0
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: while true; do
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: if [[ ! -f /etc/origin/node/node-config.yaml ]]; then
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: echo "warning: Cannot find existing node-config.yaml, waiting 15s ..." 2>&1
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: sleep 15 & wait
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: (( retries += 1 ))
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: else
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: break
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: if [[ "${retries}" -gt 40 ]]; then
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: echo "error: No existing node-config.yaml, exiting" 2>&1
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: exit 1
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: done
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: # Take over network functions on the node
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: rm -Rf /etc/cni/net.d/*
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: rm -Rf /host/opt/cni/bin/*
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: cp -Rf /opt/cni/bin/* /host/opt/cni/bin/
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: if [[ -f /etc/sysconfig/origin-node ]]; then
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: set -o allexport
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: source /etc/sysconfig/origin-node
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: # use either the bootstrapped node kubeconfig or the static configuration
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: file=/etc/origin/node/node.kubeconfig
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: if [[ ! -f "${file}" ]]; then
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: # use the static node config if it exists
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: # TODO: remove when static node configuration is no longer supported
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: for f in /etc/origin/node/system*.kubeconfig; do
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: echo "info: Using ${f} for node configuration" 1>&2
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: file="${f}"
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: break
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: done
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: # Use the same config as the node, but with the service account token
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: oc config "--config=${file}" view --flatten > /tmp/kubeconfig
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: oc config --config=/tmp/kubeconfig set-credentials sa "--token=$( cat /var/run/secrets/kubernetes.io/serviceaccount/token )"
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: oc config --config=/tmp/kubeconfig set-context "$( oc config --config=/tmp/kubeconfig current-context )" --user=sa
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: # Launch the network process
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: exec openshift start network --config=/etc/origin/node/node-config.yaml --kubeconfig=/tmp/kubeconfig --loglevel=${DEBUG_LOGLEVEL:-2}
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: ],Args:[],WorkingDir:,Ports:[{healthz 10256 10256 TCP }],Env:[{OPENSHIFT_DNS_DOMAIN cluster.local nil}],Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{cpu: {{100 -3} {<nil>} 100m DecimalSI},memory: {{209715200 0} {<nil>} BinarySI},},},VolumeMounts:[{host-config true /etc/origin/node/ <nil>} {host-sysconfig-node true /etc/sysconfig/origin-node <nil>} {host-var-run false /var/run <nil>} {host-var-run-dbus true /var/run/dbus/ <nil>} {host-var-run-ovs true /var/run/openvswitch/ <nil>} {host-var-run-kubernetes true /var/run/kubernetes/ <nil>} {host-var-run-openshift-sdn false /var/run/openshift-sdn <nil>} {host-opt-cni-bin false /host/opt/cni/bin <nil>} {host-etc-cni-netd false /etc/cni/net.d <nil>} {host-var-lib-cni-networks-openshift-sdn false /var/lib/cni/networks/openshift-sdn <nil>} {sdn-token-5rn77 true /var/run/secrets/kubernetes.io/serviceaccount <nil>}],LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:nil,Privileged:*true,SELinuxOptions:nil,RunAsUser:*0,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[],TerminationMessagePolicy:File,VolumeDevices:[],} in pod sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: I1119 08:28:26.128433 62740 kuberuntime_container.go:100] Generating ref for container sdn: &v1.ObjectReference{Kind:"Pod", Namespace:"openshift-sdn", Name:"sdn-pgmqr", UID:"40bf4573-ebd0-11e8-8b2e-5254005db2c9", APIVersion:"v1", ResourceVersion:"11367", FieldPath:"spec.containers{sdn}"}
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: I1119 08:28:26.128717 62740 kubelet_pods.go:172] container: openshift-sdn/sdn-pgmqr/sdn podIP: "192.168.122.91" creating hosts mount: true
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: I1119 08:28:26.129071 62740 server.go:435] Event(v1.ObjectReference{Kind:"Pod", Namespace:"openshift-sdn", Name:"sdn-pgmqr", UID:"40bf4573-ebd0-11e8-8b2e-5254005db2c9", APIVersion:"v1", ResourceVersion:"11367", FieldPath:"spec.containers{sdn}"}): type: 'Normal' reason: 'Pulled' Container image "docker.io/openshift/origin-node:v3.10" already present on machine
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: I1119 08:28:26.135842 62740 expiration_cache.go:98] Entry version: {key:version obj:0xc420e59860} has expired
Nov 19 08:28:26 os-master-1.example.com origin-node[62740]: I1119 08:28:26.138128 62740 docker_service.go:440] Setting cgroup parent to: "kubepods-burstable-pod40bf4573_ebd0_11e8_8b2e_5254005db2c9.slice"
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.219016 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:26.218970594 +0000 UTC m=+3283.853384776)
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.221036 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.221168 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.187392ms)
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.318922 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:26.318856884 +0000 UTC m=+3283.953271093)
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.320446 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.320548 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.682293ms)
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.419032 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:26.419009813 +0000 UTC m=+3284.053424008)
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.421954 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.422061 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (3.041122ms)
Nov 19 08:28:26 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:26.422838 40659 wrap.go:42] GET /api/v1/nodes/os-node-8.example.com?resourceVersion=0&timeout=10s: (1.71406ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.62:38670]
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.443013 40820 attach_detach_controller.go:493] processVolumesInUse for node "os-node-8.example.com"
Nov 19 08:28:26 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:26.443788 40659 wrap.go:42] PATCH /api/v1/nodes/os-node-8.example.com/status?timeout=10s: (9.142992ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.62:38670]
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.518979 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:26.518954343 +0000 UTC m=+3284.153368552)
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.520688 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.520774 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.811356ms)
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.618986 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:26.618961815 +0000 UTC m=+3284.253376020)
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.620742 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.620828 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.85686ms)
Nov 19 08:28:26 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.623517 40820 reflector.go:286] github.com/openshift/origin/vendor/k8s.io/client-go/informers/factory.go:86: forcing resync
Nov 19 08:28:27 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:26.684623 40659 wrap.go:42] GET /api/v1/namespaces/kube-system/configmaps/kube-controller-manager: (195.119003ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/leader-election] 192.168.122.91:60300]
Nov 19 08:28:27 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:26.694217 40659 wrap.go:42] PUT /api/v1/namespaces/kube-system/configmaps/kube-controller-manager: (7.514261ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/leader-election] 192.168.122.91:60300]
Nov 19 08:28:27 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:27.135793 40659 wrap.go:42] GET /api/v1/namespaces/kube-system/configmaps/openshift-master-controllers: (4.095305ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60300]
Nov 19 08:28:27 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:27.145459 40659 wrap.go:42] PUT /api/v1/namespaces/kube-system/configmaps/openshift-master-controllers: (7.011586ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60300]
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.695628 40820 leaderelection.go:199] successfully renewed lease kube-system/kube-controller-manager
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.718938 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:26.718877896 +0000 UTC m=+3284.353292092)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.720273 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.720358 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.470094ms)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.819030 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:26.819006244 +0000 UTC m=+3284.453420440)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.820475 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.820537 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.52368ms)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.918953 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:26.918922113 +0000 UTC m=+3284.553336301)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.921779 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:26.921915 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.951855ms)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.019063 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:27.019037185 +0000 UTC m=+3284.653451386)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.021418 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.021522 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.473624ms)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.118991 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:27.118957002 +0000 UTC m=+3284.753371224)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.120941 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.121063 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.093924ms)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.146773 40820 leaderelection.go:199] successfully renewed lease kube-system/openshift-master-controllers
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.218980 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:27.218953989 +0000 UTC m=+3284.853368210)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.220811 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.220920 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.919489ms)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.318925 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:27.318866298 +0000 UTC m=+3284.953280504)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.320827 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.321009 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.012013ms)
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:26.818384 62740 kubelet.go:1961] SyncLoop (housekeeping)
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.203639 62740 server.go:435] Event(v1.ObjectReference{Kind:"Pod", Namespace:"openshift-sdn", Name:"sdn-pgmqr", UID:"40bf4573-ebd0-11e8-8b2e-5254005db2c9", APIVersion:"v1", ResourceVersion:"11367", FieldPath:"spec.containers{sdn}"}): type: 'Normal' reason: 'Created' Created container
Nov 19 08:28:27 os-master-1.example.com systemd[1]: Started libcontainer container 2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c.
-- Subject: Unit docker-2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c.scope has finished start-up
-- Defined-By: systemd
-- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel
--
-- Unit docker-2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c.scope has finished starting up.
--
-- The start-up result is done.
Nov 19 08:28:27 os-master-1.example.com systemd[1]: Starting libcontainer container 2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c.
-- Subject: Unit docker-2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c.scope has begun start-up
-- Defined-By: systemd
-- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel
--
-- Unit docker-2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c.scope has begun starting up.
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.419058 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:27.419017268 +0000 UTC m=+3285.053431470)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.421199 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.421343 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.314521ms)
Nov 19 08:28:27 os-master-1.example.com oci-systemd-hook[80635]: systemdhook <debug>: 2211a3968b2c: Skipping as container command is /bin/bash, not init or systemd
Nov 19 08:28:27 os-master-1.example.com oci-umount[80636]: umounthook <debug>: prestart container_id:2211a3968b2c rootfs:/var/lib/docker/overlay2/1e7e1aa39aa2504b52cf0ded1052049c2bd8bc3b3f52d79e6e1baf04c9662c9c/merged
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.519286 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:27.519254646 +0000 UTC m=+3285.153668868)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.521270 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.521351 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.087675ms)
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.542561 62740 factory.go:113] Using factory "docker" for container "/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod40bf4573_ebd0_11e8_8b2e_5254005db2c9.slice/docker-2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c.scope"
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.547071 62740 server.go:435] Event(v1.ObjectReference{Kind:"Pod", Namespace:"openshift-sdn", Name:"sdn-pgmqr", UID:"40bf4573-ebd0-11e8-8b2e-5254005db2c9", APIVersion:"v1", ResourceVersion:"11367", FieldPath:"spec.containers{sdn}"}): type: 'Normal' reason: 'Started' Started container
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.550568 62740 manager.go:997] Added container: "/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod40bf4573_ebd0_11e8_8b2e_5254005db2c9.slice/docker-2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c.scope" (aliases: [k8s_sdn_sdn-pgmqr_openshift-sdn_40bf4573-ebd0-11e8-8b2e-5254005db2c9_11 2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c], namespace: "docker")
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.551815 62740 handler.go:325] Added event &{/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod40bf4573_ebd0_11e8_8b2e_5254005db2c9.slice/docker-2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c.scope 2018-11-19 08:28:26.145439484 +0000 UTC containerCreation {<nil>}}
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.552048 62740 container.go:448] Start housekeeping for container "/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod40bf4573_ebd0_11e8_8b2e_5254005db2c9.slice/docker-2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c.scope"
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.619008 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:27.618968111 +0000 UTC m=+3285.253382293)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.620432 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.620497 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.522492ms)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.718873 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:27.71885096 +0000 UTC m=+3285.353265161)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.721718 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.721790 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.929687ms)
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.810372 62740 generic.go:147] GenericPLEG: 40bf4573-ebd0-11e8-8b2e-5254005db2c9/2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c: non-existent -> running
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.813620 62740 kuberuntime_manager.go:852] getSandboxIDByPodUID got sandbox IDs ["5fd7fcbd62c89c72ef83f0b96462ec355dd845d0e1a221eb349a78408fac2208"] for pod "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.818834 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:27.81881587 +0000 UTC m=+3285.453230041)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.820379 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.820429 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.608493ms)
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.835359 62740 generic.go:380] PLEG: Write status for sdn-pgmqr/openshift-sdn: &container.PodStatus{ID:"40bf4573-ebd0-11e8-8b2e-5254005db2c9", Name:"sdn-pgmqr", Namespace:"openshift-sdn", IP:"", ContainerStatuses:[]*container.ContainerStatus{(*container.ContainerStatus)(0xc420d5ad20), (*container.ContainerStatus)(0xc4221bd960)}, SandboxStatuses:[]*v1alpha2.PodSandboxStatus{(*v1alpha2.PodSandboxStatus)(0xc4226fcd70)}} (err: <nil>)
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.835552 62740 kubelet.go:1919] SyncLoop (PLEG): "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)", event: &pleg.PodLifecycleEvent{ID:"40bf4573-ebd0-11e8-8b2e-5254005db2c9", Type:"ContainerStarted", Data:"2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c"}
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.835686 62740 kubelet_pods.go:1372] Generating status for "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.837008 62740 volume_manager.go:347] Waiting for volumes to attach and mount for pod "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:27 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:27.842910 40659 wrap.go:42] GET /api/v1/namespaces/openshift-sdn/pods/sdn-pgmqr: (4.742969ms) 200 [[hyperkube/v1.10.0+b81c8f8 (linux/amd64) kubernetes/b81c8f8] 192.168.122.91:35472]
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.852142 62740 config.go:297] Setting pods for source api
Nov 19 08:28:27 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:27.852280 40659 wrap.go:42] PUT /api/v1/namespaces/openshift-sdn/pods/sdn-pgmqr/status: (6.791507ms) 200 [[hyperkube/v1.10.0+b81c8f8 (linux/amd64) kubernetes/b81c8f8] 192.168.122.91:35472]
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.854457 40820 daemon_controller.go:517] Pod sdn-pgmqr updated.
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.852787 62740 status_manager.go:481] Status for pod "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)" updated successfully: (31, {Phase:Running Conditions:[{Type:Initialized Status:True LastProbeTime:0001-01-01 00:00:00 +0000 UTC LastTransitionTime:2018-11-19 07:53:51 +0000 UTC Reason: Message:} {Type:Ready Status:True LastProbeTime:0001-01-01 00:00:00 +0000 UTC LastTransitionTime:2018-11-19 08:28:27 +0000 UTC Reason: Message:} {Type:PodScheduled Status:True LastProbeTime:0001-01-01 00:00:00 +0000 UTC LastTransitionTime:2018-11-19 07:54:12 +0000 UTC Reason: Message:}] Message: Reason: NominatedNodeName: HostIP:192.168.122.91 PodIP:192.168.122.91 StartTime:2018-11-19 07:53:51 +0000 UTC InitContainerStatuses:[] ContainerStatuses:[{Name:sdn State:{Waiting:nil Running:&ContainerStateRunning{StartedAt:2018-11-19 08:28:27 +0000 UTC,} Terminated:nil} LastTerminationState:{Waiting:nil Running:nil Terminated:&ContainerStateTerminated{ExitCode:255,Signal:0,Reason:Error,Message:,StartedAt:2018-11-19 08:23:23 +0000 UTC,FinishedAt:2018-11-19 08:23:25 +0000 UTC,ContainerID:docker://d9b4320360a032613339922e004acf1b470607b6ee6f56249f2236d365e31264,}} Ready:true RestartCount:11 Image:docker.io/openshift/node:v3.10.0 ImageID:docker-pullable://docker.io/openshift/node@sha256:a6294e3d1bd6459c20e231b3276fb9dd47a8ac2db8c6a3cd258c7499e0d1d2a3 ContainerID:docker://2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c}] QOSClass:Burstable})
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.853700 62740 kubelet.go:1887] SyncLoop (RECONCILE, "api"): "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.858158 40820 controller_utils.go:176] Controller expectations fulfilled &controller.ControlleeExpectations{add:0, del:0, key:"openshift-sdn/sdn", timestamp:time.Time{wall:0xbef4bab847236b87, ext:3082754176040, loc:(*time.Location)(0xf552ba0)}}
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.858963 40820 controller_utils.go:210] Setting expectations &controller.ControlleeExpectations{add:0, del:0, key:"openshift-sdn/sdn", timestamp:time.Time{wall:0xbef4baeaf332a3f8, ext:3285493371038, loc:(*time.Location)(0xf552ba0)}}
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.859032 40820 daemon_controller.go:912] Nodes needing daemon pods for daemon set sdn: [], creating 0
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.859080 40820 daemon_controller.go:963] Pods to delete for daemon set sdn: [], deleting 0
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.859100 40820 controller_utils.go:176] Controller expectations fulfilled &controller.ControlleeExpectations{add:0, del:0, key:"openshift-sdn/sdn", timestamp:time.Time{wall:0xbef4baeaf332a3f8, ext:3285493371038, loc:(*time.Location)(0xf552ba0)}}
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.859218 40820 update.go:387] Getting unavailable numbers
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.859839 40820 update.go:426] DaemonSet openshift-sdn/sdn, maxUnavailable: 1, numUnavailable: 0
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.859860 40820 update.go:59] Marking all unavailable old pods for deletion
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.859873 40820 update.go:69] Marking old pods for deletion
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.859922 40820 controller_utils.go:210] Setting expectations &controller.ControlleeExpectations{add:0, del:0, key:"openshift-sdn/sdn", timestamp:time.Time{wall:0xbef4baeaf341497f, ext:3285494330893, loc:(*time.Location)(0xf552ba0)}}
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.859954 40820 daemon_controller.go:912] Nodes needing daemon pods for daemon set sdn: [], creating 0
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.860014 40820 daemon_controller.go:963] Pods to delete for daemon set sdn: [], deleting 0
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.860096 40820 daemon_controller.go:1028] Updating daemon set status
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.861522 40820 disruption.go:340] updatePod called on pod "sdn-pgmqr"
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.861666 40820 disruption.go:403] No PodDisruptionBudgets found for pod sdn-pgmqr, PodDisruptionBudget controller will avoid syncing.
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.861686 40820 disruption.go:343] No matching pdb for pod "sdn-pgmqr"
Nov 19 08:28:27 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:27.902640 40659 wrap.go:42] GET /api/v1/namespaces/kube-system/secrets/daemon-set-controller-token-mpfzk: (40.064953ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:27 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:27.906012 40659 wrap.go:42] GET /api/v1/namespaces/kube-system/serviceaccounts/daemon-set-controller: (2.534487ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:27 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:27.915054 40659 wrap.go:42] PUT /apis/extensions/v1beta1/namespaces/openshift-sdn/daemonsets/sdn/status: (53.288559ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/system:serviceaccount:kube-system:daemon-set-controller] 192.168.122.91:60344]
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.915940 40820 daemon_controller.go:1085] Finished syncing daemon set "openshift-sdn/sdn" (61.414485ms)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.919011 40820 daemon_controller.go:169] Updating daemon set sdn
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.919739 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:27.919727873 +0000 UTC m=+3285.554142075)
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.921184 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.921518 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.783828ms)
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.921537 62740 desired_state_of_world_populator.go:302] Added volume "host-config" (volSpec="host-config") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.921616 62740 desired_state_of_world_populator.go:302] Added volume "host-sysconfig-node" (volSpec="host-sysconfig-node") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.921678 62740 desired_state_of_world_populator.go:302] Added volume "host-modules" (volSpec="host-modules") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.921722 62740 desired_state_of_world_populator.go:302] Added volume "host-var-run" (volSpec="host-var-run") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.921758 62740 desired_state_of_world_populator.go:302] Added volume "host-var-run-dbus" (volSpec="host-var-run-dbus") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.921793 62740 desired_state_of_world_populator.go:302] Added volume "host-var-run-ovs" (volSpec="host-var-run-ovs") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.921828 62740 desired_state_of_world_populator.go:302] Added volume "host-var-run-kubernetes" (volSpec="host-var-run-kubernetes") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.921979 62740 desired_state_of_world_populator.go:302] Added volume "host-var-run-openshift-sdn" (volSpec="host-var-run-openshift-sdn") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.922020 62740 desired_state_of_world_populator.go:302] Added volume "host-opt-cni-bin" (volSpec="host-opt-cni-bin") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.922056 62740 desired_state_of_world_populator.go:302] Added volume "host-etc-cni-netd" (volSpec="host-etc-cni-netd") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.922091 62740 desired_state_of_world_populator.go:302] Added volume "host-var-lib-cni-networks-openshift-sdn" (volSpec="host-var-lib-cni-networks-openshift-sdn") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.922138 62740 desired_state_of_world_populator.go:302] Added volume "sdn-token-5rn77" (volSpec="sdn-token-5rn77") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.922687 40820 controller_utils.go:176] Controller expectations fulfilled &controller.ControlleeExpectations{add:0, del:0, key:"openshift-sdn/sdn", timestamp:time.Time{wall:0xbef4baeaf341497f, ext:3285494330893, loc:(*time.Location)(0xf552ba0)}}
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.923468 40820 controller_utils.go:210] Setting expectations &controller.ControlleeExpectations{add:0, del:0, key:"openshift-sdn/sdn", timestamp:time.Time{wall:0xbef4baeaf70adbfa, ext:3285557872897, loc:(*time.Location)(0xf552ba0)}}
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.923556 40820 daemon_controller.go:912] Nodes needing daemon pods for daemon set sdn: [], creating 0
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.923623 40820 daemon_controller.go:963] Pods to delete for daemon set sdn: [], deleting 0
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.923651 40820 controller_utils.go:176] Controller expectations fulfilled &controller.ControlleeExpectations{add:0, del:0, key:"openshift-sdn/sdn", timestamp:time.Time{wall:0xbef4baeaf70adbfa, ext:3285557872897, loc:(*time.Location)(0xf552ba0)}}
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.923784 40820 update.go:387] Getting unavailable numbers
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.924311 40820 update.go:426] DaemonSet openshift-sdn/sdn, maxUnavailable: 1, numUnavailable: 0
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.924341 40820 update.go:59] Marking all unavailable old pods for deletion
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.924359 40820 update.go:69] Marking old pods for deletion
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.924381 40820 controller_utils.go:210] Setting expectations &controller.ControlleeExpectations{add:0, del:0, key:"openshift-sdn/sdn", timestamp:time.Time{wall:0xbef4baeaf718d7b1, ext:3285558789191, loc:(*time.Location)(0xf552ba0)}}
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.924452 40820 daemon_controller.go:912] Nodes needing daemon pods for daemon set sdn: [], creating 0
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.924511 40820 daemon_controller.go:963] Pods to delete for daemon set sdn: [], deleting 0
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.924615 40820 daemon_controller.go:1028] Updating daemon set status
Nov 19 08:28:27 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:27.925221 40820 daemon_controller.go:1085] Finished syncing daemon set "openshift-sdn/sdn" (6.149479ms)
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.932717 62740 reconciler.go:237] Starting operationExecutor.MountVolume for volume "sdn-token-5rn77" (UniqueName: "kubernetes.io/secret/40bf4573-ebd0-11e8-8b2e-5254005db2c9-sdn-token-5rn77") pod "sdn-pgmqr" (UID: "40bf4573-ebd0-11e8-8b2e-5254005db2c9") Volume is already mounted to pod, but remount was requested.
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.933014 62740 secret.go:186] Setting up volume sdn-token-5rn77 for pod 40bf4573-ebd0-11e8-8b2e-5254005db2c9 at /var/lib/origin/openshift.local.volumes/pods/40bf4573-ebd0-11e8-8b2e-5254005db2c9/volumes/kubernetes.io~secret/sdn-token-5rn77
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.974187 62740 eviction_manager.go:229] eviction manager: synchronize housekeeping
Nov 19 08:28:27 os-master-1.example.com origin-node[62740]: I1119 08:28:27.978487 62740 fs.go:421] unable to determine file system type, partition mountpoint does not exist: /var/lib/docker/containers/e3a44ffeaafd0b7e23b3496f2209125bd6ff90bc35125819e020acb8b47e8f75/shm
Nov 19 08:28:28 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:28.001302 40659 wrap.go:42] GET /api/v1/namespaces/openshift-sdn/secrets/sdn-token-5rn77: (67.383745ms) 200 [[hyperkube/v1.10.0+b81c8f8 (linux/amd64) kubernetes/b81c8f8] 192.168.122.91:35472]
Nov 19 08:28:28 os-master-1.example.com origin-node[62740]: I1119 08:28:28.002162 62740 secret.go:217] Received secret openshift-sdn/sdn-token-5rn77 containing (4) pieces of data, 4115 total bytes
Nov 19 08:28:28 os-master-1.example.com origin-node[62740]: I1119 08:28:28.002754 62740 atomic_writer.go:156] pod openshift-sdn/sdn-pgmqr volume sdn-token-5rn77: no update required for target directory /var/lib/origin/openshift.local.volumes/pods/40bf4573-ebd0-11e8-8b2e-5254005db2c9/volumes/kubernetes.io~secret/sdn-token-5rn77
Nov 19 08:28:28 os-master-1.example.com origin-node[62740]: I1119 08:28:28.002808 62740 operation_generator.go:555] MountVolume.SetUp succeeded for volume "sdn-token-5rn77" (UniqueName: "kubernetes.io/secret/40bf4573-ebd0-11e8-8b2e-5254005db2c9-sdn-token-5rn77") pod "sdn-pgmqr" (UID: "40bf4573-ebd0-11e8-8b2e-5254005db2c9")
Nov 19 08:28:28 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:28.005365 40659 wrap.go:42] GET /api/v1/namespaces/kube-system/configmaps/kube-scheduler: (3.869569ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/leader-election] 192.168.122.91:60300]
Nov 19 08:28:28 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:28.013279 40659 wrap.go:42] PUT /api/v1/namespaces/kube-system/configmaps/kube-scheduler: (6.419233ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/leader-election] 192.168.122.91:60300]
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.013696 40820 leaderelection.go:199] successfully renewed lease kube-system/kube-scheduler
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.018818 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:28.018804895 +0000 UTC m=+3285.653219090)
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.020087 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.020175 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.363223ms)
Nov 19 08:28:28 os-master-1.example.com origin-node[62740]: I1119 08:28:28.044572 62740 helpers.go:840] eviction manager: observations: signal=imagefs.available, available: 64760988Ki, capacity: 72117368Ki, time: 2018-11-19 08:28:27.977835125 +0000 UTC m=+1908.041876196
Nov 19 08:28:28 os-master-1.example.com origin-node[62740]: I1119 08:28:28.044680 62740 helpers.go:840] eviction manager: observations: signal=imagefs.inodesFree, available: 4552188, capacity: 4480Ki, time: 2018-11-19 08:28:27.977835125 +0000 UTC m=+1908.041876196
Nov 19 08:28:28 os-master-1.example.com origin-node[62740]: I1119 08:28:28.044712 62740 helpers.go:840] eviction manager: observations: signal=pid.available, available: 130542, capacity: 128Ki, time: 2018-11-19 08:28:28.042523286 +0000 UTC m=+1908.106564302
Nov 19 08:28:28 os-master-1.example.com origin-node[62740]: I1119 08:28:28.044738 62740 helpers.go:840] eviction manager: observations: signal=memory.available, available: 14244536Ki, capacity: 16265308Ki, time: 2018-11-19 08:28:27.977835125 +0000 UTC m=+1908.041876196
Nov 19 08:28:28 os-master-1.example.com origin-node[62740]: I1119 08:28:28.044764 62740 helpers.go:840] eviction manager: observations: signal=allocatableMemory.available, available: 16242344Ki, capacity: 16265308Ki, time: 2018-11-19 08:28:28.04441913 +0000 UTC m=+1908.108460142
Nov 19 08:28:28 os-master-1.example.com origin-node[62740]: I1119 08:28:28.044789 62740 helpers.go:840] eviction manager: observations: signal=nodefs.available, available: 64760988Ki, capacity: 72117368Ki, time: 2018-11-19 08:28:27.977835125 +0000 UTC m=+1908.041876196
Nov 19 08:28:28 os-master-1.example.com origin-node[62740]: I1119 08:28:28.044814 62740 helpers.go:840] eviction manager: observations: signal=nodefs.inodesFree, available: 4552188, capacity: 4480Ki, time: 2018-11-19 08:28:27.977835125 +0000 UTC m=+1908.041876196
Nov 19 08:28:28 os-master-1.example.com origin-node[62740]: I1119 08:28:28.044864 62740 eviction_manager.go:336] eviction manager: no resources are starved
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.118958 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:28.118933692 +0000 UTC m=+3285.753347880)
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.120142 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.120204 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.263784ms)
Nov 19 08:28:28 os-master-1.example.com origin-node[62740]: I1119 08:28:28.137593 62740 volume_manager.go:380] All volumes are attached and mounted for pod "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:28 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:28.175451 40659 wrap.go:42] GET /api/v1/namespaces/openshift-sdn/secrets/sdn-dockercfg-fp4z5: (36.664273ms) 200 [[hyperkube/v1.10.0+b81c8f8 (linux/amd64) kubernetes/b81c8f8] 192.168.122.91:35472]
Nov 19 08:28:28 os-master-1.example.com origin-node[62740]: I1119 08:28:28.177089 62740 kuberuntime_manager.go:570] computePodActions got {KillPod:false CreateSandbox:false SandboxID:5fd7fcbd62c89c72ef83f0b96462ec355dd845d0e1a221eb349a78408fac2208 Attempt:0 NextInitContainerToStart:nil ContainersToStart:[] ContainersToKill:map[]} for pod "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.219001 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:28.218976073 +0000 UTC m=+3285.853390278)
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.220842 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.220964 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.977942ms)
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.319020 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:28.31898697 +0000 UTC m=+3285.953401171)
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.321549 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.321639 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.6425ms)
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.418908 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:28.418865852 +0000 UTC m=+3286.053280022)
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.419794 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.419833 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (962.131µs)
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.431443 40820 node_controller.go:998] Node os-master-1.example.com ReadyCondition updated. Updating timestamp.
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.431511 40820 node_controller.go:998] Node os-node-9.example.com ReadyCondition updated. Updating timestamp.
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.431543 40820 node_controller.go:998] Node os-node-3.example.com ReadyCondition updated. Updating timestamp.
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.431658 40820 node_controller.go:998] Node os-node-7.example.com ReadyCondition updated. Updating timestamp.
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.431694 40820 node_controller.go:998] Node os-node-8.example.com ReadyCondition updated. Updating timestamp.
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.518987 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:28.51895787 +0000 UTC m=+3286.153372065)
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.520365 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.520453 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.485437ms)
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.619146 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:28.619104828 +0000 UTC m=+3286.253519029)
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.621866 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.622019 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.895966ms)
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.719113 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:28.719071868 +0000 UTC m=+3286.353486064)
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.720815 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.720891 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.811643ms)
Nov 19 08:28:28 os-master-1.example.com origin-node[62740]: I1119 08:28:28.818731 62740 kubelet.go:1961] SyncLoop (housekeeping)
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.819013 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:28.818984519 +0000 UTC m=+3286.453398714)
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.820491 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.820556 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.562322ms)
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.919014 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:28.918990639 +0000 UTC m=+3286.553404831)
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.920577 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:28 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:28.920702 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.702307ms)
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.018925 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:29.018866508 +0000 UTC m=+3286.653280700)
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.020432 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.020482 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.609405ms)
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.118962 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:29.118903123 +0000 UTC m=+3286.753317305)
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.121430 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.122040 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (3.128202ms)
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.218989 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:29.218952783 +0000 UTC m=+3286.853366976)
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.221431 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.222215 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (3.251863ms)
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.318912 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:29.318861034 +0000 UTC m=+3286.953275214)
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.321435 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.322342 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (3.470358ms)
Nov 19 08:28:29 os-master-1.example.com oci-systemd-hook[80787]: systemdhook <debug>: 2211a3968b2c: Skipping as container command is /bin/bash, not init or systemd
Nov 19 08:28:29 os-master-1.example.com oci-umount[80788]: umounthook <debug>: 2211a3968b2c: only runs in prestart stage, ignoring
Nov 19 08:28:29 os-master-1.example.com dockerd-current[33852]: time="2018-11-19T08:28:29.407729674Z" level=error msg="containerd: deleting container" error="exit status 1: \"container 2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c does not exist\\none or more of the container deletions failed\\n\""
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.419059 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:29.419033035 +0000 UTC m=+3287.053447237)
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.420719 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.420790 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.747454ms)
Nov 19 08:28:29 os-master-1.example.com dockerd-current[33852]: time="2018-11-19T08:28:29.451250350Z" level=warning msg="2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c cleanup: failed to unmount secrets: invalid argument"
Nov 19 08:28:29 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.481455 40659 wrap.go:42] GET /api/v1/nodes/os-node-10.example.com?resourceVersion=0&timeout=10s: (2.114313ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.110:52148]
Nov 19 08:28:29 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.501441 40659 wrap.go:42] PATCH /api/v1/nodes/os-node-10.example.com/status?timeout=10s: (7.05063ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.110:52148]
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.502475 40820 attach_detach_controller.go:493] processVolumesInUse for node "os-node-10.example.com"
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.518983 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:29.518966941 +0000 UTC m=+3287.153381151)
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.520407 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.520495 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.520231ms)
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.619026 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:29.618997952 +0000 UTC m=+3287.253412156)
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.620671 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:29 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.620759 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.751428ms)
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.701359 40659 wrap.go:42] GET /api/v1/namespaces/kube-system/configmaps/kube-controller-manager: (4.243612ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/leader-election] 192.168.122.91:60300]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.713769 40659 wrap.go:42] PUT /api/v1/namespaces/kube-system/configmaps/kube-controller-manager: (7.222865ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/leader-election] 192.168.122.91:60300]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.837631 40659 wrap.go:42] GET /api: (942.015µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.839849 40659 wrap.go:42] GET /apis: (873.547µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.841430 40659 wrap.go:42] GET /apis/apps/v1/controllerrevisions?resourceVersion=15536&timeoutSeconds=430&watch=true: (7m10.00164992s) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/leader-election] 192.168.122.91:60300]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.842427 40659 wrap.go:42] GET /apis/apiregistration.k8s.io/v1beta1: (1.119457ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.843874 40659 get.go:238] Starting watch for /apis/apps/v1/controllerrevisions, rv=15536 labels= fields= timeout=5m29s
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.845476 40659 wrap.go:42] GET /apis/extensions/v1beta1: (1.071413ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.847682 40659 wrap.go:42] GET /apis/apps/v1: (885.151µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.849925 40659 wrap.go:42] GET /apis/apps/v1beta1: (1.038394ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.851754 40659 wrap.go:42] GET /apis/apps/v1beta2: (844.778µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.714829 40820 leaderelection.go:199] successfully renewed lease kube-system/kube-controller-manager
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.719000 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:29.718983394 +0000 UTC m=+3287.353397612)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.720441 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.720508 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.515894ms)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.819141 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:29.81907589 +0000 UTC m=+3287.453490108)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.820877 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.820988 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.901873ms)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.841974 40820 reflector.go:428] github.com/openshift/origin/vendor/k8s.io/client-go/informers/factory.go:86: Watch close - *v1.ControllerRevision total 0 items received
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.918949 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:29.918930393 +0000 UTC m=+3287.553344596)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.921358 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:29.921442 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.497355ms)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.018965 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:30.018945414 +0000 UTC m=+3287.653359631)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.020606 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.020674 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.717156ms)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.081183 40820 leaderelection.go:199] successfully renewed lease kube-system/kube-scheduler
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.118974 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:30.118954329 +0000 UTC m=+3287.753368551)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.120674 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.120752 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.787968ms)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.218975 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:30.218950174 +0000 UTC m=+3287.853364374)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.220354 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.220414 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.456032ms)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.277537 40820 daemon_controller.go:517] Pod sdn-pgmqr updated.
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.278197 40820 disruption.go:340] updatePod called on pod "sdn-pgmqr"
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.278275 40820 disruption.go:403] No PodDisruptionBudgets found for pod sdn-pgmqr, PodDisruptionBudget controller will avoid syncing.
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.278293 40820 disruption.go:343] No matching pdb for pod "sdn-pgmqr"
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.280734 40820 controller_utils.go:176] Controller expectations fulfilled &controller.ControlleeExpectations{add:0, del:0, key:"openshift-sdn/sdn", timestamp:time.Time{wall:0xbef4baeaf718d7b1, ext:3285558789191, loc:(*time.Location)(0xf552ba0)}}
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.281754 40820 controller_utils.go:210] Setting expectations &controller.ControlleeExpectations{add:0, del:0, key:"openshift-sdn/sdn", timestamp:time.Time{wall:0xbef4baeb90cb0b0d, ext:3287916156492, loc:(*time.Location)(0xf552ba0)}}
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.281814 40820 daemon_controller.go:912] Nodes needing daemon pods for daemon set sdn: [], creating 0
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.281947 40820 daemon_controller.go:963] Pods to delete for daemon set sdn: [], deleting 0
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.281987 40820 controller_utils.go:176] Controller expectations fulfilled &controller.ControlleeExpectations{add:0, del:0, key:"openshift-sdn/sdn", timestamp:time.Time{wall:0xbef4baeb90cb0b0d, ext:3287916156492, loc:(*time.Location)(0xf552ba0)}}
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.282210 40820 update.go:387] Getting unavailable numbers
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.282773 40820 update.go:426] DaemonSet openshift-sdn/sdn, maxUnavailable: 1, numUnavailable: 1
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.282807 40820 update.go:59] Marking all unavailable old pods for deletion
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.282829 40820 update.go:69] Marking old pods for deletion
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.282862 40820 controller_utils.go:210] Setting expectations &controller.ControlleeExpectations{add:0, del:0, key:"openshift-sdn/sdn", timestamp:time.Time{wall:0xbef4baeb90dc0636, ext:3287917269214, loc:(*time.Location)(0xf552ba0)}}
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.282957 40820 daemon_controller.go:912] Nodes needing daemon pods for daemon set sdn: [], creating 0
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.283061 40820 daemon_controller.go:963] Pods to delete for daemon set sdn: [], deleting 0
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.283222 40820 daemon_controller.go:1028] Updating daemon set status
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.296065 40820 daemon_controller.go:1085] Finished syncing daemon set "openshift-sdn/sdn" (18.43556ms)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.296798 40820 daemon_controller.go:169] Updating daemon set sdn
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.301038 40820 controller_utils.go:176] Controller expectations fulfilled &controller.ControlleeExpectations{add:0, del:0, key:"openshift-sdn/sdn", timestamp:time.Time{wall:0xbef4baeb90dc0636, ext:3287917269214, loc:(*time.Location)(0xf552ba0)}}
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.301789 40820 controller_utils.go:210] Setting expectations &controller.ControlleeExpectations{add:0, del:0, key:"openshift-sdn/sdn", timestamp:time.Time{wall:0xbef4baeb91fcc48c, ext:3287936192434, loc:(*time.Location)(0xf552ba0)}}
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.301846 40820 daemon_controller.go:912] Nodes needing daemon pods for daemon set sdn: [], creating 0
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.302004 40820 daemon_controller.go:963] Pods to delete for daemon set sdn: [], deleting 0
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.302044 40820 controller_utils.go:176] Controller expectations fulfilled &controller.ControlleeExpectations{add:0, del:0, key:"openshift-sdn/sdn", timestamp:time.Time{wall:0xbef4baeb91fcc48c, ext:3287936192434, loc:(*time.Location)(0xf552ba0)}}
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.302245 40820 update.go:387] Getting unavailable numbers
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.302858 40820 update.go:426] DaemonSet openshift-sdn/sdn, maxUnavailable: 1, numUnavailable: 1
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.302925 40820 update.go:59] Marking all unavailable old pods for deletion
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.302950 40820 update.go:69] Marking old pods for deletion
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.302982 40820 controller_utils.go:210] Setting expectations &controller.ControlleeExpectations{add:0, del:0, key:"openshift-sdn/sdn", timestamp:time.Time{wall:0xbef4baeb920f05ea, ext:3287937388690, loc:(*time.Location)(0xf552ba0)}}
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.303035 40820 daemon_controller.go:912] Nodes needing daemon pods for daemon set sdn: [], creating 0
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.303132 40820 daemon_controller.go:963] Pods to delete for daemon set sdn: [], deleting 0
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.303277 40820 daemon_controller.go:1028] Updating daemon set status
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.304206 40820 daemon_controller.go:1085] Finished syncing daemon set "openshift-sdn/sdn" (7.2723ms)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.318873 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:30.318856121 +0000 UTC m=+3287.953270314)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.320981 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.321040 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.176344ms)
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.863837 62740 generic.go:147] GenericPLEG: 40bf4573-ebd0-11e8-8b2e-5254005db2c9/2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c: running -> exited
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.866819 62740 kuberuntime_manager.go:852] getSandboxIDByPodUID got sandbox IDs ["5fd7fcbd62c89c72ef83f0b96462ec355dd845d0e1a221eb349a78408fac2208"] for pod "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.894190 62740 generic.go:380] PLEG: Write status for sdn-pgmqr/openshift-sdn: &container.PodStatus{ID:"40bf4573-ebd0-11e8-8b2e-5254005db2c9", Name:"sdn-pgmqr", Namespace:"openshift-sdn", IP:"", ContainerStatuses:[]*container.ContainerStatus{(*container.ContainerStatus)(0xc420718fc0), (*container.ContainerStatus)(0xc4207196c0)}, SandboxStatuses:[]*v1alpha2.PodSandboxStatus{(*v1alpha2.PodSandboxStatus)(0xc42299dc20)}} (err: <nil>)
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.894361 62740 kubelet.go:1919] SyncLoop (PLEG): "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)", event: &pleg.PodLifecycleEvent{ID:"40bf4573-ebd0-11e8-8b2e-5254005db2c9", Type:"ContainerDied", Data:"2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c"}
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.894476 62740 kubelet_pods.go:1372] Generating status for "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.894726 62740 kuberuntime_container.go:789] Removing container "d9b4320360a032613339922e004acf1b470607b6ee6f56249f2236d365e31264"
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.894727 62740 kubelet_pods.go:1372] Generating status for "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.896227 62740 volume_manager.go:347] Waiting for volumes to attach and mount for pod "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.931190 62740 desired_state_of_world_populator.go:302] Added volume "host-config" (volSpec="host-config") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.931322 62740 desired_state_of_world_populator.go:302] Added volume "host-sysconfig-node" (volSpec="host-sysconfig-node") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.931430 62740 desired_state_of_world_populator.go:302] Added volume "host-modules" (volSpec="host-modules") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.931535 62740 desired_state_of_world_populator.go:302] Added volume "host-var-run" (volSpec="host-var-run") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.931637 62740 desired_state_of_world_populator.go:302] Added volume "host-var-run-dbus" (volSpec="host-var-run-dbus") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.931782 62740 desired_state_of_world_populator.go:302] Added volume "host-var-run-ovs" (volSpec="host-var-run-ovs") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.931925 62740 desired_state_of_world_populator.go:302] Added volume "host-var-run-kubernetes" (volSpec="host-var-run-kubernetes") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.932031 62740 desired_state_of_world_populator.go:302] Added volume "host-var-run-openshift-sdn" (volSpec="host-var-run-openshift-sdn") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.932166 62740 desired_state_of_world_populator.go:302] Added volume "host-opt-cni-bin" (volSpec="host-opt-cni-bin") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.932271 62740 desired_state_of_world_populator.go:302] Added volume "host-etc-cni-netd" (volSpec="host-etc-cni-netd") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.932369 62740 desired_state_of_world_populator.go:302] Added volume "host-var-lib-cni-networks-openshift-sdn" (volSpec="host-var-lib-cni-networks-openshift-sdn") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.932478 62740 desired_state_of_world_populator.go:302] Added volume "sdn-token-5rn77" (volSpec="sdn-token-5rn77") for pod "40bf4573-ebd0-11e8-8b2e-5254005db2c9" to desired state.
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.943789 62740 reconciler.go:237] Starting operationExecutor.MountVolume for volume "sdn-token-5rn77" (UniqueName: "kubernetes.io/secret/40bf4573-ebd0-11e8-8b2e-5254005db2c9-sdn-token-5rn77") pod "sdn-pgmqr" (UID: "40bf4573-ebd0-11e8-8b2e-5254005db2c9") Volume is already mounted to pod, but remount was requested.
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.944079 62740 secret.go:186] Setting up volume sdn-token-5rn77 for pod 40bf4573-ebd0-11e8-8b2e-5254005db2c9 at /var/lib/origin/openshift.local.volumes/pods/40bf4573-ebd0-11e8-8b2e-5254005db2c9/volumes/kubernetes.io~secret/sdn-token-5rn77
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.944332 62740 secret.go:217] Received secret openshift-sdn/sdn-token-5rn77 containing (4) pieces of data, 4115 total bytes
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.945072 62740 atomic_writer.go:156] pod openshift-sdn/sdn-pgmqr volume sdn-token-5rn77: no update required for target directory /var/lib/origin/openshift.local.volumes/pods/40bf4573-ebd0-11e8-8b2e-5254005db2c9/volumes/kubernetes.io~secret/sdn-token-5rn77
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:29.945154 62740 operation_generator.go:555] MountVolume.SetUp succeeded for volume "sdn-token-5rn77" (UniqueName: "kubernetes.io/secret/40bf4573-ebd0-11e8-8b2e-5254005db2c9-sdn-token-5rn77") pod "sdn-pgmqr" (UID: "40bf4573-ebd0-11e8-8b2e-5254005db2c9")
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:30.196768 62740 volume_manager.go:380] All volumes are attached and mounted for pod "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:30.197076 62740 kuberuntime_manager.go:513] Container {Name:sdn Image:docker.io/openshift/origin-node:v3.10 Command:[/bin/bash -c #!/bin/bash
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: set -euo pipefail
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: # if another process is listening on the cni-server socket, wait until it exits
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: trap 'kill $(jobs -p); exit 0' TERM
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: retries=0
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: while true; do
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: if echo 'test' | socat - UNIX-CONNECT:/var/run/openshift-sdn/cni-server.sock >/dev/null; then
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: echo "warning: Another process is currently listening on the CNI socket, waiting 15s ..." 2>&1
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: sleep 15 & wait
Nov 19 08:28:30 os-master-1.example.com etcd[19943]: apply entries took too long [124.576692ms for 1 entries]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.853983 40659 wrap.go:42] GET /apis/events.k8s.io/v1beta1: (750.769µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.857127 40659 wrap.go:42] GET /apis/authentication.k8s.io/v1: (1.670637ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.859189 40659 wrap.go:42] GET /apis/authentication.k8s.io/v1beta1: (829.908µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.861554 40659 wrap.go:42] GET /apis/authorization.k8s.io/v1: (795.3µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.863088 40659 wrap.go:42] GET /apis/authorization.k8s.io/v1beta1: (532.645µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.864940 40659 wrap.go:42] GET /apis/autoscaling/v1: (781.183µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.866865 40659 wrap.go:42] GET /apis/autoscaling/v2beta1: (688.66µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.869183 40659 wrap.go:42] GET /apis/batch/v1: (1.096008ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.871551 40659 wrap.go:42] GET /apis/batch/v1beta1: (912.747µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.873597 40659 wrap.go:42] GET /apis/batch/v2alpha1: (792.793µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.875469 40659 wrap.go:42] GET /apis/certificates.k8s.io/v1beta1: (676.544µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: (( retries += 1 ))
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: else
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: break
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: if [[ "${retries}" -gt 40 ]]; then
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: echo "error: Another process is currently listening on the CNI socket, exiting" 2>&1
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: exit 1
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: done
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: # if the node config doesn't exist yet, wait until it does
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: retries=0
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: while true; do
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: if [[ ! -f /etc/origin/node/node-config.yaml ]]; then
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: echo "warning: Cannot find existing node-config.yaml, waiting 15s ..." 2>&1
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: sleep 15 & wait
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: (( retries += 1 ))
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: else
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: break
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: if [[ "${retries}" -gt 40 ]]; then
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: echo "error: No existing node-config.yaml, exiting" 2>&1
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: exit 1
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: done
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: # Take over network functions on the node
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: rm -Rf /etc/cni/net.d/*
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: rm -Rf /host/opt/cni/bin/*
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: cp -Rf /opt/cni/bin/* /host/opt/cni/bin/
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: if [[ -f /etc/sysconfig/origin-node ]]; then
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: set -o allexport
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: source /etc/sysconfig/origin-node
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: # use either the bootstrapped node kubeconfig or the static configuration
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: file=/etc/origin/node/node.kubeconfig
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: if [[ ! -f "${file}" ]]; then
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: # use the static node config if it exists
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: # TODO: remove when static node configuration is no longer supported
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: for f in /etc/origin/node/system*.kubeconfig; do
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: echo "info: Using ${f} for node configuration" 1>&2
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: file="${f}"
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: break
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: done
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: # Use the same config as the node, but with the service account token
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: oc config "--config=${file}" view --flatten > /tmp/kubeconfig
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: oc config --config=/tmp/kubeconfig set-credentials sa "--token=$( cat /var/run/secrets/kubernetes.io/serviceaccount/token )"
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: oc config --config=/tmp/kubeconfig set-context "$( oc config --config=/tmp/kubeconfig current-context )" --user=sa
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: # Launch the network process
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: exec openshift start network --config=/etc/origin/node/node-config.yaml --kubeconfig=/tmp/kubeconfig --loglevel=${DEBUG_LOGLEVEL:-2}
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: ] Args:[] WorkingDir: Ports:[{Name:healthz HostPort:10256 ContainerPort:10256 Protocol:TCP HostIP:}] EnvFrom:[] Env:[{Name:OPENSHIFT_DNS_DOMAIN Value:cluster.local ValueFrom:nil}] Resources:{Limits:map[] Requests:map[memory:{i:{value:209715200 scale:0} d:{Dec:<nil>} s: Format:BinarySI} cpu:{i:{value:100 scale:-3} d:{Dec:<nil>} s:100m Format:DecimalSI}]} VolumeMounts:[{Name:host-config ReadOnly:true MountPath:/etc/origin/node/ SubPath: MountPropagation:<nil>} {Name:host-sysconfig-node ReadOnly:true MountPath:/etc/sysconfig/origin-node SubPath: MountPropagation:<nil>} {Name:host-var-run ReadOnly:false MountPath:/var/run SubPath: MountPropagation:<nil>} {Name:host-var-run-dbus ReadOnly:true MountPath:/var/run/dbus/ SubPath: MountPropagation:<nil>} {Name:host-var-run-ovs ReadOnly:true MountPath:/var/run/openvswitch/ SubPath: MountPropagation:<nil>} {Name:host-var-run-kubernetes ReadOnly:true MountPath:/var/run/kubernetes/ SubPath: MountPropagation:<nil>} {Name:host-var-run-openshift-sdn ReadOnly:false MountPath:/var/run/openshift-sdn SubPath: MountPropagation:<nil>} {Name:host-opt-cni-bin ReadOnly:false MountPath:/host/opt/cni/bin SubPath: MountPropagation:<nil>} {Name:host-etc-cni-netd ReadOnly:false MountPath:/etc/cni/net.d SubPath: MountPropagation:<nil>} {Name:host-var-lib-cni-networks-openshift-sdn ReadOnly:false MountPath:/var/lib/cni/networks/openshift-sdn SubPath: MountPropagation:<nil>} {Name:sdn-token-5rn77 ReadOnly:true MountPath:/var/run/secrets/kubernetes.io/serviceaccount SubPath: MountPropagation:<nil>}] VolumeDevices:[] LivenessProbe:nil ReadinessProbe:nil Lifecycle:nil TerminationMessagePath:/dev/termination-log TerminationMessagePolicy:File ImagePullPolicy:IfNotPresent SecurityContext:&SecurityContext{Capabilities:nil,Privileged:*true,SELinuxOptions:nil,RunAsUser:*0,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,} Stdin:false StdinOnce:false TTY:false} is dead, but RestartPolicy says that we should restart it.
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:30.197185 62740 kuberuntime_manager.go:570] computePodActions got {KillPod:false CreateSandbox:false SandboxID:5fd7fcbd62c89c72ef83f0b96462ec355dd845d0e1a221eb349a78408fac2208 Attempt:0 NextInitContainerToStart:nil ContainersToStart:[0] ContainersToKill:map[]} for pod "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:30.197423 62740 kuberuntime_manager.go:757] checking backoff for container "sdn" in pod "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:30.197727 62740 kuberuntime_manager.go:767] Back-off 5m0s restarting failed container=sdn pod=sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:30.197758 62740 kuberuntime_manager.go:720] Backing Off restarting container &Container{Name:sdn,Image:docker.io/openshift/origin-node:v3.10,Command:[/bin/bash -c #!/bin/bash
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: set -euo pipefail
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: # if another process is listening on the cni-server socket, wait until it exits
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: trap 'kill $(jobs -p); exit 0' TERM
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: retries=0
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: while true; do
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: if echo 'test' | socat - UNIX-CONNECT:/var/run/openshift-sdn/cni-server.sock >/dev/null; then
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: echo "warning: Another process is currently listening on the CNI socket, waiting 15s ..." 2>&1
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: sleep 15 & wait
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: (( retries += 1 ))
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: else
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: break
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: if [[ "${retries}" -gt 40 ]]; then
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: echo "error: Another process is currently listening on the CNI socket, exiting" 2>&1
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: exit 1
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: done
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: # if the node config doesn't exist yet, wait until it does
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: retries=0
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: while true; do
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: if [[ ! -f /etc/origin/node/node-config.yaml ]]; then
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: echo "warning: Cannot find existing node-config.yaml, waiting 15s ..." 2>&1
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: sleep 15 & wait
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: (( retries += 1 ))
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: else
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: break
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: if [[ "${retries}" -gt 40 ]]; then
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: echo "error: No existing node-config.yaml, exiting" 2>&1
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: exit 1
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: done
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: # Take over network functions on the node
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: rm -Rf /etc/cni/net.d/*
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: rm -Rf /host/opt/cni/bin/*
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: cp -Rf /opt/cni/bin/* /host/opt/cni/bin/
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: if [[ -f /etc/sysconfig/origin-node ]]; then
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: set -o allexport
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: source /etc/sysconfig/origin-node
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: # use either the bootstrapped node kubeconfig or the static configuration
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: file=/etc/origin/node/node.kubeconfig
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: if [[ ! -f "${file}" ]]; then
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: # use the static node config if it exists
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: # TODO: remove when static node configuration is no longer supported
Nov 19 08:28:30 os-master-1.example.com etcd[19943]: avoid queries with large range/delete range!
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.877996 40659 wrap.go:42] GET /apis/networking.k8s.io/v1: (1.149085ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.879969 40659 wrap.go:42] GET /apis/policy/v1beta1: (758.88µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.882182 40659 wrap.go:42] GET /apis/authorization.openshift.io/v1: (1.024126ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.884127 40659 wrap.go:42] GET /apis/rbac.authorization.k8s.io/v1: (704.734µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.886186 40659 wrap.go:42] GET /apis/rbac.authorization.k8s.io/v1beta1: (696.232µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.888427 40659 wrap.go:42] GET /apis/settings.k8s.io/v1alpha1: (895.238µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.890529 40659 wrap.go:42] GET /apis/storage.k8s.io/v1: (789.133µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.892600 40659 wrap.go:42] GET /apis/storage.k8s.io/v1beta1: (826.438µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.894551 40659 wrap.go:42] GET /apis/admissionregistration.k8s.io/v1beta1: (807.961µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.896782 40659 wrap.go:42] GET /apis/apiextensions.k8s.io/v1beta1: (945.152µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.898855 40659 wrap.go:42] GET /apis/apps.openshift.io/v1: (767.537µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.418871 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:30.418852295 +0000 UTC m=+3288.053266484)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.420588 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.420660 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.799282ms)
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: for f in /etc/origin/node/system*.kubeconfig; do
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: echo "info: Using ${f} for node configuration" 1>&2
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: file="${f}"
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: break
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: done
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: fi
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: # Use the same config as the node, but with the service account token
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: oc config "--config=${file}" view --flatten > /tmp/kubeconfig
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: oc config --config=/tmp/kubeconfig set-credentials sa "--token=$( cat /var/run/secrets/kubernetes.io/serviceaccount/token )"
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: oc config --config=/tmp/kubeconfig set-context "$( oc config --config=/tmp/kubeconfig current-context )" --user=sa
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: # Launch the network process
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: exec openshift start network --config=/etc/origin/node/node-config.yaml --kubeconfig=/tmp/kubeconfig --loglevel=${DEBUG_LOGLEVEL:-2}
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: ],Args:[],WorkingDir:,Ports:[{healthz 10256 10256 TCP }],Env:[{OPENSHIFT_DNS_DOMAIN cluster.local nil}],Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{cpu: {{100 -3} {<nil>} 100m DecimalSI},memory: {{209715200 0} {<nil>} BinarySI},},},VolumeMounts:[{host-config true /etc/origin/node/ <nil>} {host-sysconfig-node true /etc/sysconfig/origin-node <nil>} {host-var-run false /var/run <nil>} {host-var-run-dbus true /var/run/dbus/ <nil>} {host-var-run-ovs true /var/run/openvswitch/ <nil>} {host-var-run-kubernetes true /var/run/kubernetes/ <nil>} {host-var-run-openshift-sdn false /var/run/openshift-sdn <nil>} {host-opt-cni-bin false /host/opt/cni/bin <nil>} {host-etc-cni-netd false /etc/cni/net.d <nil>} {host-var-lib-cni-networks-openshift-sdn false /var/lib/cni/networks/openshift-sdn <nil>} {sdn-token-5rn77 true /var/run/secrets/kubernetes.io/serviceaccount <nil>}],LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:nil,Privileged:*true,SELinuxOptions:nil,RunAsUser:*0,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[],TerminationMessagePolicy:File,VolumeDevices:[],} in pod sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: E1119 08:28:30.197973 62740 pod_workers.go:186] Error syncing pod 40bf4573-ebd0-11e8-8b2e-5254005db2c9 ("sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"), skipping: failed to "StartContainer" for "sdn" with CrashLoopBackOff: "Back-off 5m0s restarting failed container=sdn pod=sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:30.198043 62740 server.go:435] Event(v1.ObjectReference{Kind:"Pod", Namespace:"openshift-sdn", Name:"sdn-pgmqr", UID:"40bf4573-ebd0-11e8-8b2e-5254005db2c9", APIVersion:"v1", ResourceVersion:"11367", FieldPath:"spec.containers{sdn}"}): type: 'Warning' reason: 'BackOff' Back-off restarting failed container
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:30.276890 62740 config.go:297] Setting pods for source api
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:30.277846 62740 status_manager.go:481] Status for pod "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)" updated successfully: (32, {Phase:Running Conditions:[{Type:Initialized Status:True LastProbeTime:0001-01-01 00:00:00 +0000 UTC LastTransitionTime:2018-11-19 07:53:51 +0000 UTC Reason: Message:} {Type:Ready Status:False LastProbeTime:0001-01-01 00:00:00 +0000 UTC LastTransitionTime:2018-11-19 08:28:29 +0000 UTC Reason:ContainersNotReady Message:containers with unready status: [sdn]} {Type:PodScheduled Status:True LastProbeTime:0001-01-01 00:00:00 +0000 UTC LastTransitionTime:2018-11-19 07:54:12 +0000 UTC Reason: Message:}] Message: Reason: NominatedNodeName: HostIP:192.168.122.91 PodIP:192.168.122.91 StartTime:2018-11-19 07:53:51 +0000 UTC InitContainerStatuses:[] ContainerStatuses:[{Name:sdn State:{Waiting:nil Running:nil Terminated:&ContainerStateTerminated{ExitCode:255,Signal:0,Reason:Error,Message:,StartedAt:2018-11-19 08:28:27 +0000 UTC,FinishedAt:2018-11-19 08:28:29 +0000 UTC,ContainerID:docker://2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c,}} LastTerminationState:{Waiting:nil Running:nil Terminated:&ContainerStateTerminated{ExitCode:255,Signal:0,Reason:Error,Message:,StartedAt:2018-11-19 08:23:23 +0000 UTC,FinishedAt:2018-11-19 08:23:25 +0000 UTC,ContainerID:docker://d9b4320360a032613339922e004acf1b470607b6ee6f56249f2236d365e31264,}} Ready:false RestartCount:11 Image:docker.io/openshift/node:v3.10.0 ImageID:docker-pullable://docker.io/openshift/node@sha256:a6294e3d1bd6459c20e231b3276fb9dd47a8ac2db8c6a3cd258c7499e0d1d2a3 ContainerID:docker://2211a3968b2c2e0641b196dc4652c162f8e742e94f7d552c96390d994960be1c}] QOSClass:Burstable})
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:30.278132 62740 kubelet.go:1887] SyncLoop (RECONCILE, "api"): "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: W1119 08:28:30.473755 62740 cni.go:171] Unable to update cni config: No networks found in /etc/cni/net.d
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:30.474122 62740 kubelet.go:2140] Container runtime status: Runtime Conditions: RuntimeReady=true reason: message:, NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: E1119 08:28:30.474170 62740 kubelet.go:2143] Container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.900866 40659 wrap.go:42] GET /apis/build.openshift.io/v1: (783.038µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.902855 40659 wrap.go:42] GET /apis/image.openshift.io/v1: (637.202µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.905299 40659 wrap.go:42] GET /apis/network.openshift.io/v1: (843.379µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.907232 40659 wrap.go:42] GET /apis/oauth.openshift.io/v1: (887.038µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.909531 40659 wrap.go:42] GET /apis/project.openshift.io/v1: (1.08836ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.912176 40659 wrap.go:42] GET /apis/quota.openshift.io/v1: (1.001403ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.914350 40659 wrap.go:42] GET /apis/route.openshift.io/v1: (1.028879ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.916677 40659 wrap.go:42] GET /apis/security.openshift.io/v1: (843.762µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.918767 40659 wrap.go:42] GET /apis/template.openshift.io/v1: (749.309µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.921231 40659 wrap.go:42] GET /apis/user.openshift.io/v1: (987.013µs) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:29.923613 40659 wrap.go:42] GET /api/v1: (1.152203ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:30.072825 40659 wrap.go:42] GET /api/v1/namespaces/kube-system/configmaps/kube-scheduler: (57.353197ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/leader-election] 192.168.122.91:60300]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:30.074356 40659 wrap.go:42] GET /api/v1/namespaces/openshift-sdn/pods/sdn-pgmqr: (177.504523ms) 200 [[hyperkube/v1.10.0+b81c8f8 (linux/amd64) kubernetes/b81c8f8] 192.168.122.91:35472]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:30.080431 40659 wrap.go:42] PUT /api/v1/namespaces/kube-system/configmaps/kube-scheduler: (6.009346ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/leader-election] 192.168.122.91:60300]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:30.277253 40659 wrap.go:42] PUT /api/v1/namespaces/openshift-sdn/pods/sdn-pgmqr/status: (201.215524ms) 200 [[hyperkube/v1.10.0+b81c8f8 (linux/amd64) kubernetes/b81c8f8] 192.168.122.91:35472]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:30.290654 40659 wrap.go:42] GET /api/v1/namespaces/kube-system/configmaps/openshift-master-controllers: (142.634932ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60300]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:30.295344 40659 wrap.go:42] PUT /apis/extensions/v1beta1/namespaces/openshift-sdn/daemonsets/sdn/status: (10.101686ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/system:serviceaccount:kube-system:daemon-set-controller] 192.168.122.91:60344]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:30.409263 40659 wrap.go:42] GET /api/v1/nodes/os-node-2.example.com?resourceVersion=0&timeout=10s: (1.707786ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.90:35354]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:30.426582 40659 wrap.go:42] GET /api/v1/nodes/os-node-1.example.com?resourceVersion=0&timeout=10s: (1.226386ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.42:60392]
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.518952 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:30.518878856 +0000 UTC m=+3288.153293067)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.520495 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.520555 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.66903ms)
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:30.591225 40659 wrap.go:42] PATCH /api/v1/nodes/os-node-1.example.com/status?timeout=10s: (152.228002ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.42:60392]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:30.591301 40659 wrap.go:42] PATCH /api/v1/nodes/os-node-2.example.com/status?timeout=10s: (171.98062ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.90:35354]
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.592803 40820 attach_detach_controller.go:493] processVolumesInUse for node "os-node-2.example.com"
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.593091 40820 attach_detach_controller.go:493] processVolumesInUse for node "os-node-1.example.com"
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.619017 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:30.618999373 +0000 UTC m=+3288.253413575)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.620692 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.620781 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.771866ms)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.719396 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:30.719368775 +0000 UTC m=+3288.353782996)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.721135 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.721258 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.878443ms)
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:30.818548 62740 kubelet.go:1961] SyncLoop (housekeeping)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.819077 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:30.819031194 +0000 UTC m=+3288.453445395)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.820660 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.820751 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.709827ms)
Nov 19 08:28:30 os-master-1.example.com etcd[19943]: apply entries took too long [136.230814ms for 1 entries]
Nov 19 08:28:30 os-master-1.example.com etcd[19943]: avoid queries with large range/delete range!
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:30.840697 40659 trace.go:76] Trace[863430695]: "GuaranteedUpdate etcd3: *core.ConfigMap" (started: 2018-11-19 08:28:30.294870469 +0000 UTC m=+3302.857600011) (total time: 545.597477ms):
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: Trace[863430695]: [296.164517ms] [296.164517ms] initial value restored
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: Trace[863430695]: [296.74408ms] [579.563µs] Transaction prepared
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: Trace[863430695]: [545.481111ms] [248.737031ms] Transaction committed
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: Trace[863430695]: [545.597477ms] [116.366µs] END
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:30.841401 40659 trace.go:76] Trace[1237223170]: "Update /api/v1/namespaces/kube-system/configmaps/openshift-master-controllers" (started: 2018-11-19 08:28:30.29445507 +0000 UTC m=+3302.857184609) (total time: 546.845778ms):
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: Trace[1237223170]: [196.957µs] [196.957µs] About to convert to expected version
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: Trace[1237223170]: [303.556µs] [106.599µs] Conversion done
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: Trace[1237223170]: [322.001µs] [18.445µs] About to store object in database
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: Trace[1237223170]: [546.344143ms] [546.022142ms] Object stored in database
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: Trace[1237223170]: [546.384695ms] [40.552µs] Self-link added
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: Trace[1237223170]: [546.845778ms] [461.083µs] END
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:30.841696 40659 wrap.go:42] PUT /api/v1/namespaces/kube-system/configmaps/openshift-master-controllers: (548.936585ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60300]
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.842810 40820 leaderelection.go:199] successfully renewed lease kube-system/openshift-master-controllers
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:30.898488 40659 wrap.go:42] GET /api/v1/namespaces/default: (178.633437ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 127.0.0.1:60614]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:30.900723 40659 wrap.go:42] GET /api/v1/namespaces/openshift-sdn/secrets/sdn-dockercfg-fp4z5?resourceVersion=0: (85.384509ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.90:35354]
Nov 19 08:28:30 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:30.902926 40659 wrap.go:42] GET /api/v1/namespaces/default/services/kubernetes: (3.060102ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 127.0.0.1:60614]
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:30.906373 62740 generic.go:147] GenericPLEG: 40bf4573-ebd0-11e8-8b2e-5254005db2c9/d9b4320360a032613339922e004acf1b470607b6ee6f56249f2236d365e31264: exited -> non-existent
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:30.908987 62740 kuberuntime_manager.go:852] getSandboxIDByPodUID got sandbox IDs ["5fd7fcbd62c89c72ef83f0b96462ec355dd845d0e1a221eb349a78408fac2208"] for pod "sdn-pgmqr_openshift-sdn(40bf4573-ebd0-11e8-8b2e-5254005db2c9)"
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.918992 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:30.918961357 +0000 UTC m=+3288.553375538)
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.920365 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:30 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:30.920456 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.486354ms)
Nov 19 08:28:30 os-master-1.example.com origin-node[62740]: I1119 08:28:30.925989 62740 generic.go:380] PLEG: Write status for sdn-pgmqr/openshift-sdn: &container.PodStatus{ID:"40bf4573-ebd0-11e8-8b2e-5254005db2c9", Name:"sdn-pgmqr", Namespace:"openshift-sdn", IP:"", ContainerStatuses:[]*container.ContainerStatus{(*container.ContainerStatus)(0xc420719ce0)}, SandboxStatuses:[]*v1alpha2.PodSandboxStatus{(*v1alpha2.PodSandboxStatus)(0xc4216a3720)}} (err: <nil>)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.019352 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:31.01931067 +0000 UTC m=+3288.653724915)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.021963 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.022137 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.808919ms)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.119171 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:31.119143976 +0000 UTC m=+3288.753558178)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.121464 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.121652 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.479731ms)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.219129 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:31.219102811 +0000 UTC m=+3288.853517058)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.221245 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.221335 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.219621ms)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.319356 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:31.319007395 +0000 UTC m=+3288.953421613)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.321468 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.321573 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.551142ms)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.373834 40820 reflector.go:286] github.com/openshift/origin/vendor/k8s.io/client-go/informers/factory.go:86: forcing resync
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.418931 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:31.418914206 +0000 UTC m=+3289.053328408)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.420385 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.420468 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.543576ms)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.500623 40820 pv_controller_base.go:402] resyncing PV controller
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.519060 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:31.519012637 +0000 UTC m=+3289.153426847)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.520472 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.520573 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.551404ms)
Nov 19 08:28:31 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:31.543729 40659 wrap.go:42] GET /api/v1/nodes/os-node-5.example.com?resourceVersion=0&timeout=10s: (1.555185ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.133:38242]
Nov 19 08:28:31 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:31.562172 40659 wrap.go:42] PATCH /api/v1/nodes/os-node-5.example.com/status?timeout=10s: (6.874039ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.133:38242]
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.563406 40820 attach_detach_controller.go:493] processVolumesInUse for node "os-node-5.example.com"
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.619068 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:31.619006458 +0000 UTC m=+3289.253420662)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.620813 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.621428 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.409006ms)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.719091 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:31.719062863 +0000 UTC m=+3289.353477081)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.721678 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.721837 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.756077ms)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.819068 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:31.819042245 +0000 UTC m=+3289.453456493)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.820563 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.820656 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.604434ms)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.919033 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:31.918971491 +0000 UTC m=+3289.553385696)
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.920921 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:31 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:31.921084 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.101315ms)
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.019069 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:32.018974783 +0000 UTC m=+3289.653388988)
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.022557 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.022740 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (3.720317ms)
Nov 19 08:28:32 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:32.073283 40659 wrap.go:42] GET /api/v1/namespaces/kube-system/secrets/cronjob-controller-token-h6n7f: (4.689712ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:32 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:32.078553 40659 wrap.go:42] GET /api/v1/namespaces/kube-system/serviceaccounts/cronjob-controller: (4.146485ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60272]
Nov 19 08:28:32 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:32.081930 40659 wrap.go:42] GET /apis/batch/v1/jobs: (15.403823ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/system:serviceaccount:kube-system:cronjob-controller] 192.168.122.91:60344]
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.082610 40820 cronjob_controller.go:122] Found 0 jobs
Nov 19 08:28:32 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:32.085017 40659 wrap.go:42] GET /api/v1/namespaces/kube-system/configmaps/kube-scheduler: (2.765442ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/leader-election] 192.168.122.91:60300]
Nov 19 08:28:32 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:32.085869 40659 wrap.go:42] GET /apis/batch/v1beta1/cronjobs: (2.524ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/system:serviceaccount:kube-system:cronjob-controller] 192.168.122.91:60344]
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.086338 40820 cronjob_controller.go:130] Found 0 cronjobs
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.086373 40820 cronjob_controller.go:133] Found 0 groups
Nov 19 08:28:32 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:32.092307 40659 wrap.go:42] PUT /api/v1/namespaces/kube-system/configmaps/kube-scheduler: (5.693648ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/leader-election] 192.168.122.91:60300]
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.093193 40820 leaderelection.go:199] successfully renewed lease kube-system/kube-scheduler
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.119016 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:32.11897027 +0000 UTC m=+3289.753384471)
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.121473 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.121593 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.61085ms)
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.218875 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:32.218853442 +0000 UTC m=+3289.853267714)
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.221270 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.221397 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.527997ms)
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.319097 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:32.319078474 +0000 UTC m=+3289.953492679)
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.320501 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.320611 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.521505ms)
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.419035 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:32.418960601 +0000 UTC m=+3290.053374806)
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.420567 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:32 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.420811 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.838339ms)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.519369 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:32.519355104 +0000 UTC m=+3290.153769298)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.520488 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.520540 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.177839ms)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.579067 40820 attach_detach_controller.go:493] processVolumesInUse for node "os-node-6.example.com"
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.619086 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:32.619069547 +0000 UTC m=+3290.253483755)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.620516 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.620615 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.534603ms)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.718915 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:32.718864478 +0000 UTC m=+3290.353278696)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.720754 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.720852 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.976625ms)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.819109 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:32.819091127 +0000 UTC m=+3290.453505348)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.820516 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.820593 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.491723ms)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.919133 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:32.919111503 +0000 UTC m=+3290.553525704)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.921342 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:32.921426 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.300488ms)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.018996 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:33.018979733 +0000 UTC m=+3290.653393934)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.020463 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.020563 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.566419ms)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.073328 40820 leaderelection.go:199] successfully renewed lease kube-system/kube-controller-manager
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.119009 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:33.118993311 +0000 UTC m=+3290.753407503)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.120405 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.120476 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.474566ms)
Nov 19 08:28:33 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:32.558010 40659 wrap.go:42] GET /api/v1/nodes/os-node-6.example.com?resourceVersion=0&timeout=10s: (2.149659ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.61:33432]
Nov 19 08:28:33 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:32.577360 40659 wrap.go:42] PATCH /api/v1/nodes/os-node-6.example.com/status?timeout=10s: (7.096688ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.61:33432]
Nov 19 08:28:33 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:33.063295 40659 wrap.go:42] GET /api/v1/namespaces/kube-system/configmaps/kube-controller-manager: (346.803366ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/leader-election] 192.168.122.91:60300]
Nov 19 08:28:33 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:33.072298 40659 wrap.go:42] PUT /api/v1/namespaces/kube-system/configmaps/kube-controller-manager: (7.001723ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/leader-election] 192.168.122.91:60300]
Nov 19 08:28:33 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:33.269374 40659 wrap.go:42] GET /api/v1/nodes/os-node-4.example.com?resourceVersion=0&timeout=10s: (1.576473ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.241:38266]
Nov 19 08:28:33 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:33.291779 40659 wrap.go:42] PATCH /api/v1/nodes/os-node-4.example.com/status?timeout=10s: (6.677156ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.241:38266]
Nov 19 08:28:33 os-master-1.example.com origin-node[62740]: I1119 08:28:32.818356 62740 kubelet.go:1961] SyncLoop (housekeeping)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.219103 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:33.219074909 +0000 UTC m=+3290.853489137)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.220715 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.220819 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.733324ms)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.293254 40820 attach_detach_controller.go:493] processVolumesInUse for node "os-node-4.example.com"
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.319055 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:33.319030015 +0000 UTC m=+3290.953444297)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.321161 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.321265 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.223155ms)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.419239 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:33.419203125 +0000 UTC m=+3291.053617357)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.422594 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.422736 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (3.515454ms)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.432229 40820 node_controller.go:998] Node os-node-4.example.com ReadyCondition updated. Updating timestamp.
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.432517 40820 node_controller.go:998] Node os-node-5.example.com ReadyCondition updated. Updating timestamp.
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.432620 40820 node_controller.go:998] Node os-node-6.example.com ReadyCondition updated. Updating timestamp.
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.432710 40820 node_controller.go:998] Node os-node-2.example.com ReadyCondition updated. Updating timestamp.
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.432798 40820 node_controller.go:998] Node os-node-10.example.com ReadyCondition updated. Updating timestamp.
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.433020 40820 node_controller.go:998] Node os-node-1.example.com ReadyCondition updated. Updating timestamp.
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.519114 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:33.519080724 +0000 UTC m=+3291.153494929)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.520791 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.520915 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.791738ms)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.619070 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:33.619043438 +0000 UTC m=+3291.253457643)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.621110 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.621217 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.157056ms)
Nov 19 08:28:33 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:33.689440 40659 wrap.go:42] GET /api/v1/namespaces/default/pods/docker-registry-1-xrkpq: (4.682434ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/scheduler] 192.168.122.91:60300]
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.690484 40820 factory.go:1022] About to try and schedule pod docker-registry-1-xrkpq
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.690531 40820 scheduler.go:438] Attempting to schedule pod: default/docker-registry-1-xrkpq
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.691412 40820 scheduler.go:191] Failed to schedule pod: default/docker-registry-1-xrkpq
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.691510 40820 factory.go:1141] Unable to schedule default docker-registry-1-xrkpq: no fit: 0/11 nodes are available: 1 NodeNotReady, 10 CheckServiceAffinity, 10 MatchNodeSelector.; waiting
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.691696 40820 factory.go:1251] Updating pod condition for default/docker-registry-1-xrkpq to (PodScheduled==False)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.691746 40820 scheduler.go:211] Pod priority feature is not enabled. No preemption is performed.
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.692009 40820 backoff_utils.go:79] Backing off 1s
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.719050 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:33.719032757 +0000 UTC m=+3291.353446978)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.720544 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.720648 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.605473ms)
Nov 19 08:28:33 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:33.753121 40659 wrap.go:42] GET /api/v1/namespaces/openshift-node/secrets/sync-token-f67g7?resourceVersion=0: (4.617953ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.136:50920]
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.819107 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:33.819081722 +0000 UTC m=+3291.453495940)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.821587 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.821739 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.637717ms)
Nov 19 08:28:33 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:33.847749 40659 wrap.go:42] GET /api/v1/namespaces/kube-system/configmaps/openshift-master-controllers: (3.515519ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60300]
Nov 19 08:28:33 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:33.856856 40659 wrap.go:42] PUT /api/v1/namespaces/kube-system/configmaps/openshift-master-controllers: (7.253798ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.91:60300]
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.857668 40820 leaderelection.go:199] successfully renewed lease kube-system/openshift-master-controllers
Nov 19 08:28:33 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:33.900373 40659 wrap.go:42] GET /api/v1/nodes/os-node-9.example.com?resourceVersion=0&timeout=10s: (1.832051ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.179:52400]
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.919005 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:33.918985394 +0000 UTC m=+3291.553399612)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.920499 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.920624 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.627583ms)
Nov 19 08:28:33 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:33.932253 40820 attach_detach_controller.go:493] processVolumesInUse for node "os-node-9.example.com"
Nov 19 08:28:33 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:33.932835 40659 wrap.go:42] PATCH /api/v1/nodes/os-node-9.example.com/status?timeout=10s: (17.226313ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.179:52400]
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.019060 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:34.019039949 +0000 UTC m=+3291.653454170)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.020590 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.020679 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.629484ms)
Nov 19 08:28:34 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:34.030716 40659 wrap.go:42] GET /api/v1/namespaces/default/pods/router-1-5xnqc: (18.479671ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/scheduler] 192.168.122.91:60300]
Nov 19 08:28:34 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:34.030725 40659 wrap.go:42] GET /api/v1/namespaces/openshift-web-console/pods/webconsole-5f649b49b5-kldfm: (15.292084ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/scheduler] 192.168.122.91:60300]
Nov 19 08:28:34 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:34.031559 40659 wrap.go:42] GET /api/v1/namespaces/openshift-node/secrets/sync-dockercfg-g6b6k?resourceVersion=0: (65.07312ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.136:50920]
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.032198 40820 factory.go:1022] About to try and schedule pod router-1-5xnqc
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.032237 40820 scheduler.go:438] Attempting to schedule pod: default/router-1-5xnqc
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.033091 40820 scheduler.go:191] Failed to schedule pod: default/router-1-5xnqc
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.033177 40820 factory.go:1141] Unable to schedule default router-1-5xnqc: no fit: 0/11 nodes are available: 1 NodeNotReady, 10 CheckServiceAffinity, 10 MatchNodeSelector.; waiting
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.033296 40820 factory.go:1251] Updating pod condition for default/router-1-5xnqc to (PodScheduled==False)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.033329 40820 scheduler.go:211] Pod priority feature is not enabled. No preemption is performed.
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.033610 40820 factory.go:1022] About to try and schedule pod webconsole-5f649b49b5-kldfm
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.033673 40820 scheduler.go:438] Attempting to schedule pod: openshift-web-console/webconsole-5f649b49b5-kldfm
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.033869 40820 backoff_utils.go:79] Backing off 1s
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.034574 40820 scheduler.go:191] Failed to schedule pod: openshift-web-console/webconsole-5f649b49b5-kldfm
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.034681 40820 factory.go:1141] Unable to schedule openshift-web-console webconsole-5f649b49b5-kldfm: no fit: 0/11 nodes are available: 1 NodeNotReady, 10 MatchNodeSelector.; waiting
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.034855 40820 factory.go:1251] Updating pod condition for openshift-web-console/webconsole-5f649b49b5-kldfm to (PodScheduled==False)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.034960 40820 backoff_utils.go:79] Backing off 1s
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.034973 40820 scheduler.go:211] Pod priority feature is not enabled. No preemption is performed.
Nov 19 08:28:34 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:34.098589 40659 wrap.go:42] GET /api/v1/namespaces/kube-system/configmaps/kube-scheduler: (3.846855ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/leader-election] 192.168.122.91:60300]
Nov 19 08:28:34 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:34.109013 40659 wrap.go:42] PUT /api/v1/namespaces/kube-system/configmaps/kube-scheduler: (7.577022ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/leader-election] 192.168.122.91:60300]
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.109700 40820 leaderelection.go:199] successfully renewed lease kube-system/kube-scheduler
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.119047 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:34.119030854 +0000 UTC m=+3291.753445059)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.120514 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.120627 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.585261ms)
Nov 19 08:28:34 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:34.133765 40659 wrap.go:42] GET /api/v1/nodes/os-node-7.example.com?resourceVersion=0&timeout=10s: (1.290419ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.33:34028]
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.219150 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:34.219105761 +0000 UTC m=+3291.853519966)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.221084 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.221174 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.051032ms)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.319085 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:34.319055955 +0000 UTC m=+3291.953470157)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.320763 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.320868 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.800401ms)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.393102 40820 attach_detach_controller.go:493] processVolumesInUse for node "os-node-7.example.com"
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.419024 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:34.419008943 +0000 UTC m=+3292.053423148)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.420452 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.420519 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.501984ms)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.519071 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:34.519021897 +0000 UTC m=+3292.153436099)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.520777 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.520851 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.820199ms)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.619061 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:34.619033753 +0000 UTC m=+3292.253447955)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.620573 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.620642 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.598489ms)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.716009 40820 factory.go:1022] About to try and schedule pod docker-registry-1-xrkpq
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.716091 40820 scheduler.go:438] Attempting to schedule pod: default/docker-registry-1-xrkpq
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.717011 40820 scheduler.go:191] Failed to schedule pod: default/docker-registry-1-xrkpq
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.717090 40820 factory.go:1141] Unable to schedule default docker-registry-1-xrkpq: no fit: 0/11 nodes are available: 1 NodeNotReady, 10 CheckServiceAffinity, 10 MatchNodeSelector.; waiting
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.717242 40820 factory.go:1251] Updating pod condition for default/docker-registry-1-xrkpq to (PodScheduled==False)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.717280 40820 scheduler.go:211] Pod priority feature is not enabled. No preemption is performed.
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.717347 40820 backoff_utils.go:79] Backing off 2s
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.719043 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:34.719028237 +0000 UTC m=+3292.353442428)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.720263 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.720351 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.313418ms)
Nov 19 08:28:34 os-master-1.example.com etcd[19943]: apply entries took too long [215.388691ms for 1 entries]
Nov 19 08:28:34 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:34.391717 40659 wrap.go:42] PATCH /api/v1/nodes/os-node-7.example.com/status?timeout=10s: (245.840514ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.33:34028]
Nov 19 08:28:34 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:34.714518 40659 wrap.go:42] GET /api/v1/namespaces/default/pods/docker-registry-1-xrkpq: (20.780128ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/scheduler] 192.168.122.91:60300]
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.819054 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:34.819031922 +0000 UTC m=+3292.453446117)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.820669 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.820732 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.691715ms)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.919470 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:34.919438661 +0000 UTC m=+3292.553853010)
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.921473 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:34 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:34.921569 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.119505ms)
Nov 19 08:28:34 os-master-1.example.com origin-node[62740]: I1119 08:28:34.818297 62740 kubelet.go:1961] SyncLoop (housekeeping)
Nov 19 08:28:34 os-master-1.example.com etcd[19943]: avoid queries with large range/delete range!
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.019053 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:35.019032588 +0000 UTC m=+3292.653446798)
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.020758 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.020868 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.825178ms)
Nov 19 08:28:35 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:35.067738 40659 wrap.go:42] GET /api/v1/nodes/os-node-3.example.com?resourceVersion=0&timeout=10s: (1.320652ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.136:50920]
Nov 19 08:28:35 os-master-1.example.com etcd[19943]: store.index: compact 15859
Nov 19 08:28:35 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:35.090376 40659 compact.go:160] etcd: compacted rev (15859), endpoints ([https://os-master-1.example.com:2379])
Nov 19 08:28:35 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:35.091245 40659 wrap.go:42] GET /api/v1/namespaces/default/pods/router-1-5xnqc: (55.954773ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/scheduler] 192.168.122.91:60300]
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.092514 40820 factory.go:1022] About to try and schedule pod router-1-5xnqc
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.092616 40820 scheduler.go:438] Attempting to schedule pod: default/router-1-5xnqc
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.093716 40820 scheduler.go:191] Failed to schedule pod: default/router-1-5xnqc
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.093808 40820 factory.go:1141] Unable to schedule default router-1-5xnqc: no fit: 0/11 nodes are available: 1 NodeNotReady, 10 CheckServiceAffinity, 10 MatchNodeSelector.; waiting
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.094064 40820 factory.go:1251] Updating pod condition for default/router-1-5xnqc to (PodScheduled==False)
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.094114 40820 scheduler.go:211] Pod priority feature is not enabled. No preemption is performed.
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.094455 40820 backoff_utils.go:79] Backing off 2s
Nov 19 08:28:35 os-master-1.example.com etcd[19943]: finished scheduled compaction at 15859 (took 9.834147ms)
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.102596 40820 attach_detach_controller.go:493] processVolumesInUse for node "os-node-3.example.com"
Nov 19 08:28:35 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:35.114506 40659 wrap.go:42] PATCH /api/v1/nodes/os-node-3.example.com/status?timeout=10s: (34.97239ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.136:50920]
Nov 19 08:28:35 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:35.116619 40659 wrap.go:42] GET /api/v1/namespaces/openshift-web-console/pods/webconsole-5f649b49b5-kldfm: (80.472748ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc/scheduler] 192.168.122.91:60300]
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.117801 40820 factory.go:1022] About to try and schedule pod webconsole-5f649b49b5-kldfm
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.117969 40820 scheduler.go:438] Attempting to schedule pod: openshift-web-console/webconsole-5f649b49b5-kldfm
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.118799 40820 scheduler.go:191] Failed to schedule pod: openshift-web-console/webconsole-5f649b49b5-kldfm
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.118966 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:35.118946654 +0000 UTC m=+3292.753360849)
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.118972 40820 factory.go:1141] Unable to schedule openshift-web-console webconsole-5f649b49b5-kldfm: no fit: 0/11 nodes are available: 1 NodeNotReady, 10 MatchNodeSelector.; waiting
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.119111 40820 factory.go:1251] Updating pod condition for openshift-web-console/webconsole-5f649b49b5-kldfm to (PodScheduled==False)
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.119147 40820 scheduler.go:211] Pod priority feature is not enabled. No preemption is performed.
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.119322 40820 backoff_utils.go:79] Backing off 2s
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.120508 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.120578 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.622733ms)
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.219152 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:35.219117931 +0000 UTC m=+3292.853532146)
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.221397 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.221522 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.392379ms)
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.319029 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:35.319007007 +0000 UTC m=+3292.953421211)
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.320493 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.320577 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.561531ms)
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.419050 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:35.419027979 +0000 UTC m=+3293.053442181)
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.420673 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.420761 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.7237ms)
Nov 19 08:28:35 os-master-1.example.com origin-node[62740]: W1119 08:28:35.478086 62740 cni.go:171] Unable to update cni config: No networks found in /etc/cni/net.d
Nov 19 08:28:35 os-master-1.example.com origin-node[62740]: I1119 08:28:35.478801 62740 kubelet.go:2140] Container runtime status: Runtime Conditions: RuntimeReady=true reason: message:, NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
Nov 19 08:28:35 os-master-1.example.com origin-node[62740]: E1119 08:28:35.479257 62740 kubelet.go:2143] Container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.519064 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:35.519031318 +0000 UTC m=+3293.153445523)
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.521328 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.521455 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (2.412312ms)
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.619033 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:35.619015356 +0000 UTC m=+3293.253429561)
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.620628 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.620738 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.707385ms)
Nov 19 08:28:35 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:35.693229 40659 wrap.go:42] GET /api/v1/namespaces/openshift-sdn/secrets/sdn-token-5rn77?resourceVersion=0: (6.797877ms) 200 [[openshift/v1.9.1+a0ce1bc657 (linux/amd64) kubernetes/a0ce1bc] 192.168.122.179:52400]
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.719042 40820 deployment_controller.go:563] Started syncing deployment "openshift-web-console/webconsole" (2018-11-19 08:28:35.719023774 +0000 UTC m=+3293.353437975)
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.720607 40820 progress.go:189] Queueing up deployment "webconsole" for a progress check now
Nov 19 08:28:35 os-master-1.example.com origin-master-controllers[40820]: I1119 08:28:35.720698 40820 deployment_controller.go:565] Finished syncing deployment "openshift-web-console/webconsole" (1.664351ms)
Nov 19 08:28:35 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:35.766158 40659 wrap.go:42] GET /api/v1/services?resourceVersion=8008&timeoutSeconds=564&watch=true: (9m24.002148982s) 200 [[hyperkube/v1.10.0+b81c8f8 (linux/amd64) kubernetes/b81c8f8] 192.168.122.91:35472]
Nov 19 08:28:35 os-master-1.example.com origin-node[62740]: I1119 08:28:35.767215 62740 reflector.go:428] github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/kubelet.go:452: Watch close - *v1.Service total 0 items received
Nov 19 08:28:35 os-master-1.example.com atomic-openshift-master-api[40659]: I1119 08:28:35.769390 40659 get.go:238] Starting watch for /api/v1/services, rv=8008 labels= fields= timeout=7m17s
Raw
role tasks
PLAY [Update master nodes] ***********************************************************************************************************************************
TASK [Gathering Facts] ***************************************************************************************************************************************
ok: [os-master-1.example.com]
TASK [openshift_node : set_fact] *****************************************************************************************************************************
ok: [os-master-1.example.com]
TASK [openshift_node : Place openshift_kubelet_name_override file] *******************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Check for credentials file for registry auth] *****************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Create credentials for registry auth] *************************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Create credentials for registry auth (alternative)] ***********************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Setup ro mount of /root/.docker for containerized hosts] ******************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Check that node image is present] *****************************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Pre-pull node image] ******************************************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Check that pod image is present] ******************************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : pre-pull pod image] *******************************************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : update package meta data to speed install later.] *************************************************************************************
[WARNING]: Consider using the yum module rather than running yum. If you need to use command because yum is insufficient you can add warn=False to this
command task or set command_warnings=False in ansible.cfg to get rid of this message.
changed: [os-master-1.example.com]
TASK [openshift_node : Check Docker image count] ************************************************************************************************************$
skipping: [os-master-1.example.com]
TASK [openshift_node : debug] *******************************************************************************************************************************$
skipping: [os-master-1.example.com]
TASK [openshift_node : download docker upgrade rpm] *********************************************************************************************************$
skipping: [os-master-1.example.com]
TASK [openshift_node : Stage cri-o updates] *****************************************************************************************************************$
skipping: [os-master-1.example.com]
TASK [openshift_node : download new node packages] **********************************************************************************************************$
changed: [os-master-1.example.com]
TASK [openshift_node : Check status of node image pre-pull] *************************************************************************************************$
skipping: [os-master-1.example.com]
TASK [openshift_node : Check status of node pod image pre-pull] *********************************************************************************************$
changed: [os-master-1.example.com]
TASK [openshift_node : Copy node container image to ostree storage] *****************************************************************************************$
skipping: [os-master-1.example.com]
TASK [openshift_node : Stop node services] ******************************************************************************************************************$
changed: [os-master-1.example.com] => (item=origin-node)
TASK [openshift_node : Stop openvswitch service] ************************************************************************************************************$
changed: [os-master-1.example.com]
TASK [openshift_node : service] *****************************************************************************************************************************$
skipping: [os-master-1.example.com]
TASK [openshift_node : Stop crio] ***************************************************************************************************************************$
skipping: [os-master-1.example.com]
TASK [openshift_node : Clean up cri-o pods] *****************************************************************************************************************$
skipping: [os-master-1.example.com]
TASK [openshift_node : install docker upgrade rpm] ***********************************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Check if cri-o is running as a system container] **************************************************************************************
ok: [os-master-1.example.com]
TASK [openshift_node : Uninstall cri-o system container] *****************************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : List cri-o atomic image] **************************************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Delete cri-o system container image] **************************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Ensure cri-o is updated] **************************************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Remove CRI-O default configuration files] *********************************************************************************************
skipping: [os-master-1.example.com] => (item=/etc/cni/net.d/200-loopback.conf)
skipping: [os-master-1.example.com] => (item=/etc/cni/net.d/100-crio-bridge.conf)
TASK [openshift_node : install new node packages] ************************************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Setting sebool container_manage_cgroup] ***********************************************************************************************
ok: [os-master-1.example.com]
TASK [openshift_node : include_tasks] ************************************************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Update oreg value] ********************************************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Ensure the node static pod directory exists] ******************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Ensure docker-registry directory exists] **********************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Update the docker-registry CA symlink] ************************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Update node-config for static pods] ***************************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Uninstall openvswitch] ****************************************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Remove old service information] *******************************************************************************************************
ok: [os-master-1.example.com] => (item=/etc/systemd/system/docker.service.d/docker-sdn-ovs.conf)
ok: [os-master-1.example.com] => (item=/etc/systemd/system/atomic-openshift-node-dep.service)
ok: [os-master-1.example.com] => (item=/etc/systemd/system/origin-node-dep.service)
TASK [openshift_node : Remove openvswitch service information] ***********************************************************************************************
ok: [os-master-1.example.com]
TASK [openshift_node : Check for existing node-config.yaml] **************************************************************************************************
ok: [os-master-1.example.com]
TASK [openshift_node : Copy existing configuration to bootstrap configuration] *******************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Find existing credentials] ************************************************************************************************************
ok: [os-master-1.example.com]
TASK [openshift_node : Copy existing credentials to bootstrap credentials] ***********************************************************************************
changed: [os-master-1.example.com] => (item=/etc/origin/node/system:node:os-master-1.example.com.kubeconfig)
TASK [openshift_node : Remove non-bootstrap configuration] ***************************************************************************************************
ok: [os-master-1.example.com] => (item=/etc/origin/node/node.kubeconfig)
changed: [os-master-1.example.com] => (item=/etc/origin/node/node-config.yaml)
TASK [openshift_node : Update node-config to prepare for bootstrapping] **************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Use the admin.kubeconfig for the kubelet bootstrap identity] **************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Update symlink master CA for docker-registry (name changed)] *************************************************************************$
changed: [os-master-1.example.com] => (item=/etc/origin/node/client-ca.crt)
TASK [openshift_node : Remove previous bootstrap certificates] ***********************************************************************************************
ok: [os-master-1.example.com]
TASK [openshift_node : Update the sysconfig to group "ccp-openshift-master"] *********************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Reset selinux context] ****************************************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Copy node script to the node] *********************************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Install Node service file] ************************************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Ensure old system path is set] ********************************************************************************************************
skipping: [os-master-1.example.com] => (item=/etc/origin/openvswitch)
skipping: [os-master-1.example.com] => (item=/var/lib/kubelet)
skipping: [os-master-1.example.com] => (item=/opt/cni/bin)
TASK [openshift_node : Install or Update node system container] **********************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Restart network manager to ensure networking configuration is in place] ***************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Configure Node settings] **************************************************************************************************************
changed: [os-master-1.example.com] => (item={u'regex': u'^OPTIONS=', u'line': u'OPTIONS='})
changed: [os-master-1.example.com] => (item={u'regex': u'^DEBUG_LOGLEVEL=', u'line': u'DEBUG_LOGLEVEL=4'})
changed: [os-master-1.example.com] => (item={u'regex': u'^IMAGE_VERSION=', u'line': u'IMAGE_VERSION=v3.10'})
TASK [openshift_node : openshift_facts] **********************************************************************************************************************
ok: [os-master-1.example.com]
TASK [openshift_node : Configure Proxy Settings] *************************************************************************************************************
skipping: [os-master-1.example.com] => (item={u'regex': u'^HTTP_PROXY=', u'line': u'HTTP_PROXY='})
skipping: [os-master-1.example.com] => (item={u'regex': u'^HTTPS_PROXY=', u'line': u'HTTPS_PROXY='})
skipping: [os-master-1.example.com] => (item={u'regex': u'^NO_PROXY=', u'line': u'NO_PROXY=[],172.30.0.0/16,10.128.0.0/14'})
TASK [openshift_node : stat] *********************************************************************************************************************************
ok: [os-master-1.example.com]
TASK [openshift_node : Update cri-o pause image] *************************************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Check for NetworkManager service] *****************************************************************************************************
ok: [os-master-1.example.com]
TASK [openshift_node : Set fact using_network_manager] *******************************************************************************************************
ok: [os-master-1.example.com]
TASK [openshift_node : Install dnsmasq] **********************************************************************************************************************
ok: [os-master-1.example.com]
TASK [openshift_node : ensure origin/node directory exists] **************************************************************************************************
ok: [os-master-1.example.com] => (item=/etc/origin)
ok: [os-master-1.example.com] => (item=/etc/origin/node)
TASK [openshift_node : Install NetworkManager during node_bootstrap provisioning] ****************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Install network manager dispatch script] **********************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Add network manager configuration] ****************************************************************************************************
skipping: [os-master-1.example.com] => (item={u'key': u'dns', u'value': u'none', u'present': False})
RUNNING HANDLER [openshift_node : reload systemd units] ******************************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Install dnsmasq configuration] ********************************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Deploy additional dnsmasq.conf] *******************************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Create override dnsmasq systemd unit directory] ***************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Copy systemd dnsmasq unit override file] **********************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Enable dnsmasq] ***********************************************************************************************************************
ok: [os-master-1.example.com]
TASK [openshift_node : Install network manager dispatch script] **********************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Add network manager configuration] ****************************************************************************************************
ok: [os-master-1.example.com] => (item={u'key': u'dns', u'value': u'none', u'present': False})
RUNNING HANDLER [openshift_node : reload systemd units] ******************************************************************************************************
skipping: [os-master-1.example.com]
RUNNING HANDLER [openshift_node : restart NetworkManager] ****************************************************************************************************
skipping: [os-master-1.example.com]
RUNNING HANDLER [openshift_node : restart dnsmasq] ***********************************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : ensure dnsmasq is always restarted] ***************************************************************************************************
changed: [os-master-1.example.com] => {
"msg": "Restarting dnsmasq"
}
RUNNING HANDLER [openshift_node : reload systemd units] ******************************************************************************************************
skipping: [os-master-1.example.com]
RUNNING HANDLER [openshift_node : restart dnsmasq] ***********************************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Reload systemd to ensure latest unit files] *******************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Restart support services] *************************************************************************************************************
ok: [os-master-1.example.com] => (item=NetworkManager)
ok: [os-master-1.example.com] => (item=dnsmasq)
TASK [openshift_node : Restart container runtime] ************************************************************************************************************
ok: [os-master-1.example.com]
TASK [openshift_node : Restart cri-o] ************************************************************************************************************************
skipping: [os-master-1.example.com]
TASK [openshift_node : Start node service] *******************************************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Approve the node] *********************************************************************************************************************
changed: [os-master-1.example.com -> os-master-1.example.com]
TASK [openshift_node : Check status of node service] *********************************************************************************************************
changed: [os-master-1.example.com]
TASK [openshift_node : Wait for master API to come back online] **********************************************************************************************
ok: [os-master-1.example.com]
TASK [openshift_node : Wait for node to be ready] ************************************************************************************************************
FAILED - RETRYING: Wait for node to be ready (36 retries left).
FAILED - RETRYING: Wait for node to be ready (35 retries left).
FAILED - RETRYING: Wait for node to be ready (34 retries left).
FAILED - RETRYING: Wait for node to be ready (33 retries left).
FAILED - RETRYING: Wait for node to be ready (32 retries left).
FAILED - RETRYING: Wait for node to be ready (31 retries left).
FAILED - RETRYING: Wait for node to be ready (30 retries left).
FAILED - RETRYING: Wait for node to be ready (29 retries left).
FAILED - RETRYING: Wait for node to be ready (28 retries left).
FAILED - RETRYING: Wait for node to be ready (27 retries left).
FAILED - RETRYING: Wait for node to be ready (26 retries left).
FAILED - RETRYING: Wait for node to be ready (25 retries left).
FAILED - RETRYING: Wait for node to be ready (24 retries left).
FAILED - RETRYING: Wait for node to be ready (23 retries left).
FAILED - RETRYING: Wait for node to be ready (22 retries left).
FAILED - RETRYING: Wait for node to be ready (21 retries left).
FAILED - RETRYING: Wait for node to be ready (20 retries left).
FAILED - RETRYING: Wait for node to be ready (19 retries left).
FAILED - RETRYING: Wait for node to be ready (18 retries left).
FAILED - RETRYING: Wait for node to be ready (17 retries left).
FAILED - RETRYING: Wait for node to be ready (16 retries left).
FAILED - RETRYING: Wait for node to be ready (15 retries left).
FAILED - RETRYING: Wait for node to be ready (14 retries left).
FAILED - RETRYING: Wait for node to be ready (13 retries left).
FAILED - RETRYING: Wait for node to be ready (12 retries left).
FAILED - RETRYING: Wait for node to be ready (11 retries left).
FAILED - RETRYING: Wait for node to be ready (10 retries left).
FAILED - RETRYING: Wait for node to be ready (9 retries left).
FAILED - RETRYING: Wait for node to be ready (8 retries left).
FAILED - RETRYING: Wait for node to be ready (7 retries left).
FAILED - RETRYING: Wait for node to be ready (6 retries left).
FAILED - RETRYING: Wait for node to be ready (5 retries left).
FAILED - RETRYING: Wait for node to be ready (4 retries left).
FAILED - RETRYING: Wait for node to be ready (3 retries left).
FAILED - RETRYING: Wait for node to be ready (2 retries left).
FAILED - RETRYING: Wait for node to be ready (1 retries left).
fatal: [os-master-1.example.com -> os-master-1.example.com]: FAILED! => {"attempts": 36, "changed": false, "results": {"cmd": "/usr/bin/oc get node os-master-
1.example.com -o json -n default", "results": [{"apiVersion": "v1", "kind": "Node", "metadata": {"annotations": {"volumes.kubernetes.io/controller-managed-att
ach-detach": "true"}, "creationTimestamp": "2018-11-19T06:54:42Z", "labels": {"beta.kubernetes.io/arch": "amd64", "beta.kubernetes.io/os": "linux", "kubernete
s.io/hostname": "os-master-1.example.com", "node-role.kubernetes.io/master": "true", "node-type": "metrics", "purpose": "infra", "region": "infra", "size": "l
arge1", "zone": "default"}, "name": "os-master-1.example.com", "resourceVersion": "12280", "selfLink": "/api/v1/nodes/os-master-1.example.com", "uid": "fdcf93
0a-ebc7-11e8-856e-5254005db2c9"}, "spec": {"externalID": "os-master-1.example.com"}, "status": {"addresses": [{"address": "192.168.122.91", "type": "InternalI
P"}, {"address": "os-master-1.example.com", "type": "Hostname"}], "allocatable": {"cpu": "16", "hugepages-2Mi": "0", "memory": "16162908Ki", "pods": "250"}, "
capacity": {"cpu": "16", "hugepages-2Mi": "0", "memory": "16265308Ki", "pods": "250"}, "conditions": [{"lastHeartbeatTime": "2018-11-19T08:00:43Z", "lastTrans
itionTime": "2018-11-19T07:56:40Z", "message": "kubelet has sufficient disk space available", "reason": "KubeletHasSufficientDisk", "status": "False", "type":
"OutOfDisk"}, {"lastHeartbeatTime": "2018-11-19T08:00:43Z", "lastTransitionTime": "2018-11-19T07:56:40Z", "message": "kubelet has sufficient memory available
", "reason": "KubeletHasSufficientMemory", "status": "False", "type": "MemoryPressure"}, {"lastHeartbeatTime": "2018-11-19T08:00:43Z", "lastTransitionTime": "
2018-11-19T07:56:40Z", "message": "kubelet has no disk pressure", "reason": "KubeletHasNoDiskPressure", "status": "False", "type": "DiskPressure"}, {"lastHear
tbeatTime": "2018-11-19T08:00:43Z", "lastTransitionTime": "2018-11-19T07:56:40Z", "message": "runtime network not ready: NetworkReady=false reason:NetworkPlug
inNotReady message:docker: network plugin is not ready: cni config uninitialized", "reason": "KubeletNotReady", "status": "False", "type": "Ready"}, {"lastHea
rtbeatTime": "2018-11-19T08:00:43Z", "lastTransitionTime": "2018-11-19T07:56:40Z", "message": "kubelet has sufficient PID available", "reason": "KubeletHasSuf
ficientPID", "status": "False", "type": "PIDPressure"}], "daemonEndpoints": {"kubeletEndpoint": {"Port": 10250}}, "images": [{"names": ["docker.io/openshift/o
rigin-haproxy-router@sha256:8f2ecdd9b0dc99b22d8f274970933ca205cc2252a0623db9657154493135949d", "docker.io/openshift/origin-haproxy-router:v3.9.0"], "sizeBytes
": 1284810579}, {"names": ["docker.io/openshift/node@sha256:a6294e3d1bd6459c20e231b3276fb9dd47a8ac2db8c6a3cd258c7499e0d1d2a3", "docker.io/openshift/origin-nod
e@sha256:a6294e3d1bd6459c20e231b3276fb9dd47a8ac2db8c6a3cd258c7499e0d1d2a3", "docker.io/openshift/node:v3.10.0", "docker.io/openshift/origin-node:v3.10"], "siz
eBytes": 1272276032}, {"names": ["docker.io/openshift/origin-deployer@sha256:59ad668b7ba2a216d89c88f83e39a8564c8e2056cdb5931a2bf9aafb6af3dd99", "docker.io/ope
nshift/origin-deployer:v3.9.0"], "sizeBytes": 1261176213}, {"names": ["docker.io/openshift/origin-web-console@sha256:3e68a21afb90a66e1e8fcc4ac31272d397d5e6af1
88fdb9431d4d19a60ae5298", "docker.io/openshift/origin-web-console:v3.9.0"], "sizeBytes": 495221706}, {"names": ["docker.io/openshift/origin-docker-registry@sh
a256:4e0f264808067e1c20ae16acc07820374cfe68c1279cfab20e7abdaa5b5ba617", "docker.io/openshift/origin-docker-registry:v3.9.0"], "sizeBytes": 465026624}, {"names
": ["docker.io/openshift/origin-pod@sha256:2ffeb4d71a80922b9e62698100eef40284976385cf1f1b332edc8e921d48f4f5", "docker.io/openshift/origin-pod:v3.10"], "sizeBy
tes": 223970697}, {"names": ["docker.io/openshift/origin-pod@sha256:38e2dcbe2edfa202c5aabbdd00932678602962ca05499195fab8adba7dc22c16", "docker.io/openshift/or
igin-pod:v3.9.0"], "sizeBytes": 222604299}], "nodeInfo": {"architecture": "amd64", "bootID": "17d37f8e-dca5-4d08-bf72-a90d92ce2652", "containerRuntimeVersion"
: "docker://1.13.1", "kernelVersion": "3.10.0-862.14.4.el7.x86_64", "kubeProxyVersion": "v1.10.0+b81c8f8", "kubeletVersion": "v1.10.0+b81c8f8", "machineID": "
5fef7650fcc442af93ea9d76d9f95c8e", "operatingSystem": "linux", "osImage": "CentOS Linux 7 (Core)", "systemUUID": "5FEF7650-FCC4-42AF-93EA-9D76D9F95C8E"}}}], "
returncode": 0}, "state": "list"}
to retry, use: --limit @/usr/share/ansible/openshift-ansible/playbooks/byo/openshift-cluster/upgrades/v3_10/upgrade_control_plane.retry
PLAY RECAP ***************************************************************************************************************************************************
localhost : ok=13 changed=0 unreachable=0 failed=0
os-master-1.example.com : ok=258 changed=66 unreachable=0 failed=1
os-node-1.example.com : ok=22 changed=0 unreachable=0 failed=0
os-node-10.example.com : ok=22 changed=0 unreachable=0 failed=0
os-node-2.example.com : ok=22 changed=0 unreachable=0 failed=0
os-node-3.example.com : ok=22 changed=0 unreachable=0 failed=0
os-node-4.example.com : ok=22 changed=0 unreachable=0 failed=0
os-node-5.example.com : ok=22 changed=0 unreachable=0 failed=0
os-node-6.example.com : ok=22 changed=0 unreachable=0 failed=0
os-node-7.example.com : ok=22 changed=0 unreachable=0 failed=0
os-node-8.example.com : ok=22 changed=0 unreachable=0 failed=0
os-node-9.example.com : ok=22 changed=0 unreachable=0 failed=0
Failure summary:
1. Hosts: os-master-1.example.com
Play: Update master nodes
Task: Wait for node to be ready
Message: Failed without returning a message.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment