libovsdb not connected

gistfile1.txt

[trozet@fedora Downloads]$ cat ovnkube-master-not-connected.log | egrep 'reconnect|not connected|e2e-test-default-rbac-policy-gr8fr'
I1109 13:59:24.195903       1 namespace.go:183] [e2e-test-default-rbac-policy-gr8fr] adding namespace
I1109 13:59:24.920563       1 client.go:723]  "msg"="transacting operations"  "database"="OVN_Northbound" "operations"="[{Op:insert Table:Address_Set Row:map[external_ids:{GoMap:map[name:e2e-test-default-rbac-policy-gr8fr_v4]} name:a9121939770815115361] Rows:[] Columns:[] Mutations:[] Timeout:0 Where:[] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]"
I1109 13:59:24.931687       1 cache.go:642] cache "msg"="creating row" "database"="OVN_Northbound" "table"="Address_Set" "uuid"="c4fbbe02-0892-4f5e-8c25-e31f8d457188" "model"="&{UUID:c4fbbe02-0892-4f5e-8c25-e31f8d457188 Addresses:[] ExternalIDs:map[name:e2e-test-default-rbac-policy-gr8fr_v4] Name:a9121939770815115361}"
I1109 13:59:24.947273       1 address_set.go:365] New(c4fbbe02-0892-4f5e-8c25-e31f8d457188/e2e-test-default-rbac-policy-gr8fr_v4/a9121939770815115361) with []
I1109 13:59:24.947360       1 namespace.go:187] [e2e-test-default-rbac-policy-gr8fr] adding namespace took 751.426398ms
I1109 13:59:24.947405       1 namespace.go:256] [e2e-test-default-rbac-policy-gr8fr] updating namespace
I1109 13:59:24.947439       1 namespace.go:256] [e2e-test-default-rbac-policy-gr8fr] updating namespace
I1109 13:59:28.734238       1 namespace.go:256] [e2e-test-default-rbac-policy-gr8fr] updating namespace
I1109 14:00:01.337354       1 namespace.go:256] [e2e-test-default-rbac-policy-gr8fr] updating namespace
I1109 14:00:01.354491       1 namespace.go:360] [e2e-test-default-rbac-policy-gr8fr] deleting namespace
2021-11-09T14:00:22.679Z|00007|reconnect|INFO|ssl:10.0.181.18:9641: connection attempt timed out
2021-11-09T14:00:22.679Z|00008|reconnect|INFO|ssl:10.0.155.136:9641: connecting...
2021-11-09T14:00:22.788Z|00009|reconnect|INFO|ssl:10.0.155.136:9641: connected
2021-11-09T14:00:22.859Z|00011|reconnect|INFO|ssl:10.0.155.136:9641: connection attempt timed out
2021-11-09T14:00:22.859Z|00012|reconnect|INFO|ssl:10.0.206.53:9641: connecting...
I1109 14:00:22.886937       1 client.go:484] [OVN_Northbound] disconnected from ssl:10.0.181.18:9641; reconnecting ... 
2021-11-09T14:00:22.912Z|00013|reconnect|INFO|ssl:10.0.206.53:9641: connected
I1109 14:00:22.894571       1 client.go:981]  "msg"="endpoint lost leader, reconnecting" "database"="OVN_Northbound" "endpoint"="ssl:10.0.181.18:9641"
I1109 14:00:23.024108       1 namespace.go:525] Finishing deferred deletion of AddressSet for NS e2e-test-default-rbac-policy-gr8fr
I1109 14:00:23.024118       1 address_set.go:575] destroy(c4fbbe02-0892-4f5e-8c25-e31f8d457188/e2e-test-default-rbac-policy-gr8fr_v4/a9121939770815115361)
I1109 14:00:23.523751       1 client.go:681]  "msg"="blocking transaction until reconnected" "database"="OVN_Northbound" "operations"="[{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:delete Value:{GoSet:[10.129.2.115]}}] Timeout:0 Where:[where column _uuid == {68541cc1-90a3-4062-8fd1-e3e93294e61f}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]"
I1109 14:00:23.523798       1 client.go:981]  "msg"="endpoint lost leader, reconnecting" "database"="OVN_Northbound" "endpoint"="ssl:10.0.181.18:9641"
I1109 14:00:23.523840       1 client.go:981]  "msg"="endpoint lost leader, reconnecting" "database"="OVN_Northbound" "endpoint"="ssl:10.0.181.18:9641"
E1109 14:00:23.523904       1 namespace.go:527] Failed to delete AddressSet for NS e2e-test-default-rbac-policy-gr8fr: failed to destroy address set e2e-test-default-rbac-policy-gr8fr_v4 (error in transact with ops [{Op:delete Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[] Timeout:0 Where:[where column _uuid == {c4fbbe02-0892-4f5e-8c25-e31f8d457188}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
E1109 14:00:23.524028       1 ovn.go:625] failed to AddIPs to the v4 set: failed add ips to address set e2e-test-router-scoped-jktnx_v4 (error in transact with ops [{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:insert Value:{GoSet:[10.129.2.134]}}] Timeout:0 Where:[where column _uuid == {d0b2239c-ffbf-4c90-97f9-c0ee49322dee}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
E1109 14:00:23.533648       1 pods.go:139] failed to DeleteIPs to the v4 set: failed to delete ips from address set e2e-test-cli-deployment-22hxz_v4 (error in transact with ops [{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:delete Value:{GoSet:[10.129.2.110]}}] Timeout:0 Where:[where column _uuid == {68541cc1-90a3-4062-8fd1-e3e93294e61f}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
E1109 14:00:23.533671       1 namespace.go:527] Failed to delete AddressSet for NS e2e-test-cli-deployment-dlmwl: failed to destroy address set e2e-test-cli-deployment-dlmwl_v4 (error in transact with ops [{Op:delete Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[] Timeout:0 Where:[where column _uuid == {78b55641-f6a8-4275-9f1a-0f0016405c52}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
E1109 14:00:23.546078       1 ovn.go:625] failed to AddIPs to the v4 set: failed add ips to address set e2e-test-router-scoped-bw2mv_v4 (error in transact with ops [{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:insert Value:{GoSet:[10.131.0.62]}}] Timeout:0 Where:[where column _uuid == {24784343-7495-4d20-a2a4-03e3a938dac1}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
E1109 14:00:23.550672       1 ovn.go:625] failed to AddIPs to the v4 set: failed add ips to address set e2e-test-oauth-ldap-idp-chgkt_v4 (error in transact with ops [{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:insert Value:{GoSet:[10.129.2.135]}}] Timeout:0 Where:[where column _uuid == {c1a29a71-f1f6-4071-8091-9cb8f2457b4f}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
E1109 14:00:23.533765       1 namespace.go:527] Failed to delete AddressSet for NS e2e-test-cli-bnh2z: failed to destroy address set e2e-test-cli-bnh2z_v4 (error in transact with ops [{Op:delete Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[] Timeout:0 Where:[where column _uuid == {ae06514b-33fe-458d-b08d-c2d3c81fcbc3}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
E1109 14:00:23.533782       1 namespace.go:527] Failed to delete AddressSet for NS e2e-test-scopes-wffxh: failed to destroy address set e2e-test-scopes-wffxh_v4 (error in transact with ops [{Op:delete Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[] Timeout:0 Where:[where column _uuid == {305cc6d2-c46b-4c30-b836-d63544c72a57}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
E1109 14:00:23.559270       1 ovn.go:625] failed to AddIPs to the v4 set: failed add ips to address set openshift-multus_v4 (error in transact with ops [{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:insert Value:{GoSet:[10.131.0.63]}}] Timeout:0 Where:[where column _uuid == {53099903-c161-4ff0-a949-8e4456c9009b}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
E1109 14:00:23.533837       1 pods.go:144] cannot delete logical switch port e2e-test-oc-builds-52pz8_database-1-deploy, error in transact with ops [{Op:mutate Table:Logical_Switch Row:map[] Rows:[] Columns:[] Mutations:[{Column:ports Mutator:delete Value:{GoSet:[{GoUUID:778760f5-9e3d-4ffe-8b51-91a203aa23a9}]}}] Timeout:0 Where:[where column _uuid == {1eed87b8-66bd-4de8-925d-9b9c78e4e866}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:} {Op:delete Table:Logical_Switch_Port Row:map[] Rows:[] Columns:[] Mutations:[] Timeout:0 Where:[where column _uuid == {778760f5-9e3d-4ffe-8b51-91a203aa23a9}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected
I1109 14:00:23.560774       1 client.go:681]  "msg"="blocking transaction until reconnected" "database"="OVN_Northbound" "operations"="[{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:delete Value:{GoSet:[10.129.2.128]}}] Timeout:0 Where:[where column _uuid == {6f2bcb91-e46e-4d68-9efc-75af09407a9c}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]"
E1109 14:00:23.533910       1 pods.go:139] failed to DeleteIPs to the v4 set: failed to delete ips from address set e2e-test-build-webhooks-zbq6v_v4 (error in transact with ops [{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:delete Value:{GoSet:[10.131.0.51]}}] Timeout:0 Where:[where column _uuid == {fe257feb-d311-44fc-9120-f7e79359d24d}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
E1109 14:00:23.561786       1 ovn.go:625] failed to AddIPs to the v4 set: failed add ips to address set e2e-test-router-scoped-bw2mv_v4 (error in transact with ops [{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:insert Value:{GoSet:[10.128.3.176]}}] Timeout:0 Where:[where column _uuid == {24784343-7495-4d20-a2a4-03e3a938dac1}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
E1109 14:00:23.533957       1 pods.go:139] failed to DeleteIPs to the v4 set: failed to delete ips from address set e2e-test-build-webhooks-zbq6v_v4 (error in transact with ops [{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:delete Value:{GoSet:[10.128.3.172]}}] Timeout:0 Where:[where column _uuid == {fe257feb-d311-44fc-9120-f7e79359d24d}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
E1109 14:00:23.563764       1 ovn.go:625] failed to AddIPs to the v4 set: failed add ips to address set e2e-test-router-headers-vm8bx_v4 (error in transact with ops [{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:insert Value:{GoSet:[10.131.0.64]}}] Timeout:0 Where:[where column _uuid == {66d0d77c-57c8-4c6c-8583-6b0616083358}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
E1109 14:00:23.590270       1 ovn.go:625] failed to AddIPs to the v4 set: failed add ips to address set e2e-test-router-reencrypt-qmtls_v4 (error in transact with ops [{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:insert Value:{GoSet:[10.129.2.136]}}] Timeout:0 Where:[where column _uuid == {cc0de3aa-05b7-4ed9-9bd4-6256e0525e0c}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
I1109 14:00:23.534039       1 client.go:1048]  "msg"="connection lost, reconnecting" "database"="OVN_Northbound" "endpoint"="ssl:10.0.181.18:9641"
I1109 14:00:23.535552       1 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"e2e-test-router-scoped-jktnx", Name:"endpoint-1", UID:"3c9b9df0-3d72-4df4-8fca-e4e4ffd096e2", APIVersion:"v1", ResourceVersion:"221455", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingLogicalPort' failed to AddIPs to the v4 set: failed add ips to address set e2e-test-router-scoped-jktnx_v4 (error in transact with ops [{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:insert Value:{GoSet:[10.129.2.134]}}] Timeout:0 Where:[where column _uuid == {d0b2239c-ffbf-4c90-97f9-c0ee49322dee}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
I1109 14:00:23.593725       1 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"e2e-test-router-scoped-bw2mv", Name:"execpod", UID:"ed10c208-5cd7-4164-bc3e-b3074178deba", APIVersion:"v1", ResourceVersion:"221625", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingLogicalPort' failed to AddIPs to the v4 set: failed add ips to address set e2e-test-router-scoped-bw2mv_v4 (error in transact with ops [{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:insert Value:{GoSet:[10.131.0.62]}}] Timeout:0 Where:[where column _uuid == {24784343-7495-4d20-a2a4-03e3a938dac1}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
I1109 14:00:23.628675       1 client.go:497] [OVN_Northbound] reconnected to ssl:10.0.206.53:9641 after 0 retries.
I1109 14:00:23.595551       1 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"e2e-test-oauth-ldap-idp-chgkt", Name:"runonce-ldapsearch-pod", UID:"05687b5e-71a8-4ca6-8b6c-64fca85a591e", APIVersion:"v1", ResourceVersion:"221063", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingLogicalPort' failed to AddIPs to the v4 set: failed add ips to address set e2e-test-oauth-ldap-idp-chgkt_v4 (error in transact with ops [{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:insert Value:{GoSet:[10.129.2.135]}}] Timeout:0 Where:[where column _uuid == {c1a29a71-f1f6-4071-8091-9cb8f2457b4f}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
I1109 14:00:23.628708       1 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"openshift-multus", Name:"ip-reconciler-27274440--1-872df", UID:"4cabde27-ad60-4074-b4c4-6ff7aa6bbcc1", APIVersion:"v1", ResourceVersion:"221208", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingLogicalPort' failed to AddIPs to the v4 set: failed add ips to address set openshift-multus_v4 (error in transact with ops [{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:insert Value:{GoSet:[10.131.0.63]}}] Timeout:0 Where:[where column _uuid == {53099903-c161-4ff0-a949-8e4456c9009b}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
I1109 14:00:23.629163       1 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"e2e-test-router-scoped-bw2mv", Name:"router-scoped", UID:"60cf0622-8f5b-47cd-8c4f-15b8c2aa49a5", APIVersion:"v1", ResourceVersion:"221613", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingLogicalPort' failed to AddIPs to the v4 set: failed add ips to address set e2e-test-router-scoped-bw2mv_v4 (error in transact with ops [{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:insert Value:{GoSet:[10.128.3.176]}}] Timeout:0 Where:[where column _uuid == {24784343-7495-4d20-a2a4-03e3a938dac1}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
I1109 14:00:23.629176       1 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"e2e-test-router-headers-vm8bx", Name:"execpod", UID:"5ceaf564-2cf3-4719-8aca-ec200e136ee6", APIVersion:"v1", ResourceVersion:"221443", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingLogicalPort' failed to AddIPs to the v4 set: failed add ips to address set e2e-test-router-headers-vm8bx_v4 (error in transact with ops [{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:insert Value:{GoSet:[10.131.0.64]}}] Timeout:0 Where:[where column _uuid == {66d0d77c-57c8-4c6c-8583-6b0616083358}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
I1109 14:00:23.629195       1 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"e2e-test-router-reencrypt-qmtls", Name:"execpod", UID:"47af1334-cb70-4027-b204-e6425351815e", APIVersion:"v1", ResourceVersion:"221360", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingLogicalPort' failed to AddIPs to the v4 set: failed add ips to address set e2e-test-router-reencrypt-qmtls_v4 (error in transact with ops [{Op:mutate Table:Address_Set Row:map[] Rows:[] Columns:[] Mutations:[{Column:addresses Mutator:insert Value:{GoSet:[10.129.2.136]}}] Timeout:0 Where:[where column _uuid == {cc0de3aa-05b7-4ed9-9bd4-6256e0525e0c}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]: not connected)
I1109 14:00:23.882428       1 client.go:241]  "msg"="reconnected - restarting monitors" "database"="OVN_Northbound" 
I1109 14:00:25.038252       1 cache.go:632] cache "msg"="creating row" "database"="OVN_Northbound" "table"="Address_Set" "uuid"="c4fbbe02-0892-4f5e-8c25-e31f8d457188" "model"="&{UUID:c4fbbe02-0892-4f5e-8c25-e31f8d457188 Addresses:[] ExternalIDs:map[name:e2e-test-default-rbac-policy-gr8fr_v4] Name:a9121939770815115361}"
I1109 14:00:25.340737       1 client.go:981]  "msg"="endpoint lost leader, reconnecting" "database"="OVN_Northbound" "endpoint"="ssl:10.0.206.53:9641"