Created
October 26, 2022 17:14
-
-
Save sedflix/95bc34ee4a4fcde98ae93993708c864e to your computer and use it in GitHub Desktop.
calico-issue
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| # the node doesn't exist | |
| k get nodes | grep ip-10-111-38-8.ap-south-1.compute.internal | |
| <blank> | |
| calico-kube-controllers log: | |
| 022-10-26 17:06:56.985 [INFO][1] ipam.go 1174: Calico Node referenced in IPAM data does not exist error=resource does not exist: Node(<node-name>) with error: nodes "<node-name>" not found | |
| ... | |
| ... | |
| │ 2022-10-26 17:09:13.843 [INFO][1] ipam_block_reader_writer.go 334: Block must be empty but is not empty, refusing to remove affinity. host="<node-name>" subnet="subnet/cidr" │ | |
| │ 2022-10-26 17:09:13.882 [ERROR][1] ipam.go 1101: Failed to release block affinities for node calicoNode="<node-anme>" error=block 'subnet/cidr' is not empty | |
| --- | |
| # blockaffinity does exist | |
| kubectl get blockaffinity | grepip-10-111-38-8.ap-south-1.compute.internal | |
| ip-10-111-38-8.ap-south-1.compute.internal-192-168-24-64-26 31h | |
| kubectl get blockaffinity ip-10-111-38-8.ap-south-1.compute.internal-192-168-24-64-26 -o yaml | |
| apiVersion: crd.projectcalico.org/v1 | |
| kind: BlockAffinity | |
| metadata: | |
| annotations: | |
| projectcalico.org/metadata: '{"creationTimestamp":null}' | |
| creationTimestamp: "2022-10-25T09:04:35Z" | |
| generation: 2 | |
| name: ip-10-111-38-8.ap-south-1.compute.internal-192-168-24-64-26 | |
| spec: | |
| cidr: 192.168.24.64/26 | |
| deleted: "false" | |
| node: ip-10-111-38-8.ap-south-1.compute.internal | |
| state: confirmed | |
| --- | |
| # ipamblocks does exist | |
| k get ipamblocks | grep 192-168-24-64-26 | |
| 192-168-24-64-26 32h | |
| k get ipamblocks 192-168-24-64-26 -o yaml | |
| apiVersion: crd.projectcalico.org/v1 | |
| kind: IPAMBlock | |
| metadata: | |
| annotations: | |
| projectcalico.org/metadata: '{"creationTimestamp":null}' | |
| creationTimestamp: "2022-10-25T09:04:35Z" | |
| generation: 626 | |
| name: 192-168-24-64-26 | |
| spec: | |
| affinity: host:ip-10-111-38-8.ap-south-1.compute.internal | |
| allocations: | |
| - null | |
| - null | |
| - null | |
| - null | |
| - null | |
| - null | |
| - 16 | |
| - null | |
| - null | |
| - null | |
| - null | |
| - null | |
| - null | |
| - null | |
| - null | |
| - null | |
| - null | |
| - null | |
| - null | |
| - null | |
| - null | |
| - null | |
| - 22 | |
| - null | |
| - 19 | |
| - 0 | |
| - 1 | |
| - 2 | |
| - 3 | |
| - 4 | |
| - 18 | |
| - 5 | |
| - 6 | |
| - null | |
| - null | |
| - 9 | |
| - 7 | |
| - 8 | |
| - null | |
| - null | |
| - 10 | |
| - null | |
| - 17 | |
| - 15 | |
| - null | |
| - null | |
| - null | |
| - 21 | |
| - null | |
| - 20 | |
| - null | |
| - null | |
| - null | |
| - 11 | |
| - null | |
| - null | |
| - null | |
| - null | |
| - 12 | |
| - 13 | |
| - null | |
| - 14 | |
| - null | |
| - null | |
| attributes: | |
| - handle_id: vxlan-tunnel-addr-ip-10-111-16-37.ap-south-1.compute.internal | |
| secondary: | |
| node: ip-10-111-16-37.ap-south-1.compute.internal | |
| type: vxlanTunnelAddress | |
| - handle_id: k8s-pod-network.1acc3b6bb01b28fdf45a3bd2f05d11f69d18450d92eeb87cffa78802dd0db5f0 | |
| secondary: | |
| namespace: istio-system | |
| node: ip-10-111-16-37.ap-south-1.compute.internal | |
| pod: istio-cni-node-nnk78 | |
| timestamp: 2022-10-25 16:18:50.545831941 +0000 UTC | |
| - handle_id: k8s-pod-network.fb6c43ede7cdab0e627529e7bc1030f516f87a3fe0888a194f42b6176731d826 | |
| secondary: | |
| namespace: zone-logging | |
| node: ip-10-111-16-37.ap-south-1.compute.internal | |
| pod: olympus-zone-logging-zeta-shipper-gbk5w | |
| timestamp: 2022-10-25 16:18:50.59412689 +0000 UTC | |
| - handle_id: k8s-pod-network.0ac38916a38872a27796828647c21236117bfb48ee8c4b23405373ff25226c85 | |
| secondary: | |
| namespace: zone-infra | |
| node: ip-10-111-16-37.ap-south-1.compute.internal | |
| pod: olympus-zone-infra-spot-termination-exporter-96bks | |
| timestamp: 2022-10-25 16:18:51.372508533 +0000 UTC | |
| - handle_id: k8s-pod-network.dac42b0dbe3b7b0aff3d6d1029148cbc88608afa4e0e4aca3544cf65f31b2378 | |
| secondary: | |
| namespace: aura | |
| node: ip-10-111-16-37.ap-south-1.compute.internal | |
| pod: redis-cluster-server-2 | |
| timestamp: 2022-10-25 16:29:50.058404441 +0000 UTC | |
| - handle_id: k8s-pod-network.794feeefa2bae45f34904d49ca328ed8d7ad0d6a1200dbf4fac7b6fb6c478184 | |
| secondary: | |
| namespace: luminosrewards | |
| node: ip-10-111-16-37.ap-south-1.compute.internal | |
| pod: himalia-decision-engine-crud-85d59b7cc6-67vvs | |
| timestamp: 2022-10-25 16:33:39.849329372 +0000 UTC | |
| - handle_id: k8s-pod-network.8a0217fb57b1d63a3177c205af9dbadfe01ad4e53360a2ff02819f776a2ec4a1 | |
| secondary: | |
| namespace: castle | |
| node: ip-10-111-16-37.ap-south-1.compute.internal | |
| pod: castle-heap-dump-pusher-s3-qndqk | |
| timestamp: 2022-10-25 16:36:05.351072321 +0000 UTC | |
| - handle_id: k8s-pod-network.42dca1c03aea01deec541d30930d33d61809744bd4dd0715b12793c56ff76dee | |
| secondary: | |
| namespace: zone-infra | |
| node: ip-10-111-30-12.ap-south-1.compute.internal | |
| pod: olympus-zone-infra-spot-termination-exporter-v6zwj | |
| timestamp: 2022-10-26 07:51:22.53198364 +0000 UTC | |
| - handle_id: vxlan-tunnel-addr-ip-10-111-30-12.ap-south-1.compute.internal | |
| secondary: | |
| node: ip-10-111-30-12.ap-south-1.compute.internal | |
| type: vxlanTunnelAddress | |
| - handle_id: k8s-pod-network.ec2f78e2bff1176dfb307027f1c6c5f8bdf35732e0f5e0dd58fe1b5b8fbbfd67 | |
| secondary: | |
| namespace: istio-system | |
| node: ip-10-111-30-12.ap-south-1.compute.internal | |
| pod: istio-cni-node-8mqz5 | |
| timestamp: 2022-10-26 07:51:23.734741707 +0000 UTC | |
| - handle_id: k8s-pod-network.c3933818adde3f0d83628360e3e49d950183678603f69c6f214b5ed633dadc4a | |
| secondary: | |
| namespace: zone-logging | |
| node: ip-10-111-30-12.ap-south-1.compute.internal | |
| pod: olympus-zone-logging-zeta-shipper-hct6f | |
| timestamp: 2022-10-26 07:51:23.775085487 +0000 UTC | |
| - handle_id: k8s-pod-network.ea1c7667aa0c841056d28f6b77643e002e40ed79828c47dddaaea13e51ab05c5 | |
| secondary: | |
| namespace: castle | |
| node: ip-10-111-30-12.ap-south-1.compute.internal | |
| pod: castle-heap-dump-pusher-s3-94qm6 | |
| timestamp: 2022-10-26 07:51:37.310854519 +0000 UTC | |
| - handle_id: k8s-pod-network.5b6f2261ed30fbfdaf5470071f039b4956c02c5b97e7d9d674712d9132733303 | |
| secondary: | |
| namespace: aura | |
| node: ip-10-111-30-12.ap-south-1.compute.internal | |
| pod: principalliquidationworker-flinkcluster-jobmanager-0 | |
| timestamp: 2022-10-26 07:52:29.003594998 +0000 UTC | |
| - handle_id: k8s-pod-network.d31d3302703a5c267609413dfded5c360c3d204a14a4356a9cd9874338c4b261 | |
| secondary: | |
| namespace: aura | |
| node: ip-10-111-30-12.ap-south-1.compute.internal | |
| pod: emeraldaccrualworker-flinkcluster-jobmanager-0 | |
| timestamp: 2022-10-26 07:52:30.120010975 +0000 UTC | |
| - handle_id: k8s-pod-network.f845450af5a5ad04e0a3283d39b7d5e86193672d5bdaf98c8ef4cd0ba2b8ed0f | |
| secondary: | |
| namespace: jupiter | |
| node: ip-10-111-30-12.ap-south-1.compute.internal | |
| pod: locker-5bddc6ccc6-mhx6r | |
| timestamp: 2022-10-26 07:53:43.121975446 +0000 UTC | |
| - handle_id: k8s-pod-network.02abb40ab66f6384748b07c6f942fa010bf1e421edaf77d5b6132d7bda64f373 | |
| secondary: | |
| namespace: watch | |
| node: ip-10-111-30-12.ap-south-1.compute.internal | |
| pod: sparta-84f984dfc7-tzgb8 | |
| timestamp: 2022-10-26 08:03:58.8936858 +0000 UTC | |
| - handle_id: k8s-pod-network.89e02dfcc61d197f4ddc52741bbfbe11911710a51cb9c90e999ac70685921944 | |
| secondary: | |
| namespace: plutus-perf | |
| node: ip-10-111-30-12.ap-south-1.compute.internal | |
| pod: redis-cluster-haproxy-6489d68df-mkwjm | |
| timestamp: 2022-10-26 08:12:09.079594891 +0000 UTC | |
| - handle_id: k8s-pod-network.2d73ab6c70adf658fc28031374d8236a332e761884abe47a3382ce4ccb73551f | |
| secondary: | |
| namespace: athena-pci | |
| node: ip-10-111-30-12.ap-south-1.compute.internal | |
| pod: sparta-dashboard-89f9995bf-cvcbl | |
| timestamp: 2022-10-26 08:12:09.389434864 +0000 UTC | |
| - handle_id: k8s-pod-network.e90b34c39c9659aab83964b148405e5a535f3904aaec3718d8a8636a2402244d | |
| secondary: | |
| namespace: turbo | |
| node: ip-10-111-30-12.ap-south-1.compute.internal | |
| pod: hdfcturboonboarding-ganymede-shared-postgrest-768954c8bc-sjhn8 | |
| timestamp: 2022-10-26 08:12:10.305185964 +0000 UTC | |
| - handle_id: k8s-pod-network.3bf925fe750637f07ed51d97fb0820019d3dfafbe5fb705d23856d6c5bdafb37 | |
| secondary: | |
| namespace: emerald | |
| node: ip-10-111-30-12.ap-south-1.compute.internal | |
| pod: emerald-ganymede-shared-postgrest-7c67cfd586-rkxrl | |
| timestamp: 2022-10-26 08:12:11.573716327 +0000 UTC | |
| - handle_id: k8s-pod-network.f6778051df515c5b73ef9b728381d7ae1929b9ca9911949625cd91b5c0f9503d | |
| secondary: | |
| namespace: cipher | |
| node: ip-10-111-30-12.ap-south-1.compute.internal | |
| pod: nginx-s3-proxy-658bb44658-hxgxk | |
| timestamp: 2022-10-26 08:12:12.907402535 +0000 UTC | |
| - handle_id: k8s-pod-network.bd559b0a17c5a3345bbf98633255ffe26e3dbc501337605a47b0fd0ec0fe02da | |
| secondary: | |
| namespace: aura | |
| node: ip-10-111-30-12.ap-south-1.compute.internal | |
| pod: failuresimulatorworker-flinkcluster-jobmanager-0 | |
| timestamp: 2022-10-26 08:12:30.610191586 +0000 UTC | |
| - handle_id: k8s-pod-network.9bb3f4fbb7c2b066a3667bf383d54d2c7c4ed7c877f751653ba0ada23dfe656b | |
| secondary: | |
| namespace: athena-pci | |
| node: ip-10-111-30-12.ap-south-1.compute.internal | |
| pod: redis-cluster-server-2 | |
| timestamp: 2022-10-26 08:36:21.639501729 +0000 UTC | |
| cidr: 192.168.24.64/26 | |
| deleted: false | |
| sequenceNumber: 0 | |
| strictAffinity: false | |
| unallocated: | |
| - 1 | |
| - 41 | |
| - 38 | |
| - 20 | |
| - 11 | |
| - 62 | |
| - 44 | |
| - 10 | |
| - 9 | |
| - 13 | |
| - 34 | |
| - 21 | |
| - 17 | |
| - 5 | |
| - 19 | |
| - 45 | |
| - 56 | |
| - 4 | |
| - 51 | |
| - 39 | |
| - 3 | |
| - 7 | |
| - 55 | |
| - 57 | |
| - 15 | |
| - 16 | |
| - 0 | |
| - 2 | |
| - 23 | |
| - 12 | |
| - 46 | |
| - 60 | |
| - 33 | |
| - 52 | |
| - 48 | |
| - 18 | |
| - 50 | |
| - 14 | |
| - 63 | |
| - 8 | |
| - 54 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment