Skip to content

Instantly share code, notes, and snippets.

@MrHohn

MrHohn/gist:447a74e7fa96cca6481fde8d35696b9b

Created June 7, 2021 18:32
Show Gist options
  • Save MrHohn/447a74e7fa96cca6481fde8d35696b9b to your computer and use it in GitHub Desktop.
Save MrHohn/447a74e7fa96cca6481fde8d35696b9b to your computer and use it in GitHub Desktop.
Download ZIP
Raw
gistfile1.txt
I0607 16:33:15.391267 10 gce_loadbalancer.go:137] EnsureLoadBalancer(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1): ensure EXTERNAL loadbalancer
I0607 16:33:15.573965 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:33:15.391275173 +0000 UTC m=+4774.123372976 [59m59.817298447s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = <nil>, googleapi: Error 404: The resource 'projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3' was not found, notFound
I0607 16:33:15.752599 10 gce_loadbalancer_external.go:74] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc), us-west1, , [TCP/80], [bootstrap-e2e-master bootstrap-e2e-minion-group-5p5s bootstrap-e2e-minion-group-gp1b bootstrap-e2e-minion-group-rrsg], map[cloud.google.com/network-tier:Standard])
I0607 16:33:15.752620 10 gce_loadbalancer_external.go:83] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Desired network tier "Standard".
I0607 16:33:15.922996 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:33:15.752630093 +0000 UTC m=+4774.484727899 [59m59.829619122s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = <nil>, googleapi: Error 404: The resource 'projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3' was not found, notFound
I0607 16:33:16.123728 10 gen.go:2667] GCEAddresses.Get(context.Background.WithDeadline(2021-06-07 17:33:15.923064752 +0000 UTC m=+4774.655162550 [59m59.799324377s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = <nil>, googleapi: Error 404: The resource 'projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/addresses/a8d573ae414fd4ec099dff43893319d3' was not found, notFound
I0607 16:33:16.303914 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:33:16.123808663 +0000 UTC m=+4774.855906447 [59m59.819861585s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = <nil>, googleapi: Error 404: The resource 'projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3' was not found, notFound
I0607 16:33:16.304011 10 gce_loadbalancer_external.go:97] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Forwarding rule a8d573ae414fd4ec099dff43893319d3 doesn't exist.
I0607 16:33:18.122040 10 gen.go:2743] GCEAddresses.Insert(context.Background.WithDeadline(2021-06-07 17:33:16.304020365 +0000 UTC m=+4775.036118173 [59m58.181968634s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}, &{Address: AddressType: CreationTimestamp: Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Id:0 IpVersion: Kind: Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PrefixLength:0 Purpose: Region: SelfLink: Status: Subnetwork: Users:[] ServerResponse:{HTTPStatusCode:0 Header:map[]} ForceSendFields:[] NullFields:[]}) = <nil>
I0607 16:33:18.322505 10 gen.go:2667] GCEAddresses.Get(context.Background.WithDeadline(2021-06-07 17:33:18.122120826 +0000 UTC m=+4776.854218615 [59m59.799605973s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = &{Address:35.212.229.111 AddressType:EXTERNAL CreationTimestamp:2021-06-07T09:33:16.670-07:00 Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Id:6791993841211929763 IpVersion: Kind:compute#address Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PrefixLength:0 Purpose: Region:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1 SelfLink:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/addresses/a8d573ae414fd4ec099dff43893319d3 Status:RESERVED Subnetwork: Users:[] ServerResponse:{HTTPStatusCode:200 Header:map[Cache-Control:[private] Content-Type:[application/json; charset=UTF-8] Date:[Mon, 07 Jun 2021 16:33:18 GMT] Etag:[0N6j9SeZh5zZb_a742BTsAdZVIw=/-32Ak-gur5BRHFQ21XwBCGV9mqA=] Server:[ESF] Vary:[Origin X-Origin Referer] X-Content-Type-Options:[nosniff] X-Frame-Options:[SAMEORIGIN] X-Xss-Protection:[0]]} ForceSendFields:[] NullFields:[]}, <nil>
I0607 16:33:18.322646 10 gce_loadbalancer_external.go:160] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Ensured IP address 35.212.229.111 (tier: Standard).
I0607 16:33:18.541934 10 gen.go:8601] GCEFirewalls.Get(context.Background.WithDeadline(2021-06-07 17:33:18.322663792 +0000 UTC m=+4777.054761598 [59m59.780716996s]), Key{"k8s-fw-a8d573ae414fd4ec099dff43893319d3"}) = <nil>, googleapi: Error 404: The resource 'projects/k8s-jkns-gci-gce-ingress-1-6/global/firewalls/k8s-fw-a8d573ae414fd4ec099dff43893319d3' was not found, notFound
I0607 16:33:18.542024 10 gce_loadbalancer_external.go:194] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Creating firewall.
I0607 16:33:22.431094 10 gen.go:8677] GCEFirewalls.Insert(context.Background.WithDeadline(2021-06-07 17:33:18.542043279 +0000 UTC m=+4777.274141088 [59m56.11094006s]), Key{"k8s-fw-a8d573ae414fd4ec099dff43893319d3"}, &{Allowed:[0xc0038c9860] CreationTimestamp: Denied:[] Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc", "kubernetes.io/service-ip":"35.212.229.111"} DestinationRanges:[] Direction: Disabled:false Id:0 Kind: LogConfig:<nil> Name:k8s-fw-a8d573ae414fd4ec099dff43893319d3 Network:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/global/networks/bootstrap-e2e Priority:0 SelfLink: SourceRanges:[0.0.0.0/0] SourceServiceAccounts:[] SourceTags:[] TargetServiceAccounts:[] TargetTags:[bootstrap-e2e-minion] ServerResponse:{HTTPStatusCode:0 Header:map[]} ForceSendFields:[] NullFields:[]}) = <nil>
I0607 16:33:22.431189 10 gce_loadbalancer_external.go:198] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Created firewall.
I0607 16:33:22.600419 10 gen.go:28054] GCETargetPools.Get(context.Background.WithDeadline(2021-06-07 17:33:22.4311958 +0000 UTC m=+4781.163293598 [59m59.830766907s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = <nil>, googleapi: Error 404: The resource 'projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3' was not found, notFound
I0607 16:33:22.600515 10 gce_loadbalancer_external.go:207] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Target pool for service doesn't exist.
I0607 16:33:22.707880 10 gen.go:13488] GCEHttpHealthChecks.Get(context.Background.WithDeadline(2021-06-07 17:33:22.600522032 +0000 UTC m=+4781.332619834 [59m59.892632276s]), Key{"a8d573ae414fd4ec099dff43893319d3"}) = <nil>, googleapi: Error 404: The resource 'projects/k8s-jkns-gci-gce-ingress-1-6/global/httpHealthChecks/a8d573ae414fd4ec099dff43893319d3' was not found, notFound
I0607 16:33:22.707938 10 gce_loadbalancer_external.go:231] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Service needs nodes health checks.
I0607 16:33:31.513274 10 gce_loadbalancer_external.go:558] Creating targetpool a8d573ae414fd4ec099dff43893319d3 with 1 healthchecks
I0607 16:33:34.388866 10 gen.go:28130] GCETargetPools.Insert(context.Background.WithDeadline(2021-06-07 17:33:31.513279832 +0000 UTC m=+4790.245377634 [59m57.124400957s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}, &{BackupPool: CreationTimestamp: Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} FailoverRatio:0 HealthChecks:[https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/global/httpHealthChecks/k8s-fde405acdccef8bb-node] Id:0 Instances:[/zones/us-west1-b/instances/bootstrap-e2e-master /zones/us-west1-b/instances/bootstrap-e2e-minion-group-5p5s /zones/us-west1-b/instances/bootstrap-e2e-minion-group-gp1b /zones/us-west1-b/instances/bootstrap-e2e-minion-group-rrsg] Kind: Name:a8d573ae414fd4ec099dff43893319d3 Region: SelfLink: SessionAffinity:NONE ServerResponse:{HTTPStatusCode:0 Header:map[]} ForceSendFields:[] NullFields:[]}) = <nil>
I0607 16:33:34.388951 10 gce_loadbalancer_external.go:500] ensureTargetPoolAndHealthCheck(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Created health checks k8s-fde405acdccef8bb-node.
I0607 16:33:34.388960 10 gce_loadbalancer_external.go:503] ensureTargetPoolAndHealthCheck(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Created target pool.
I0607 16:33:34.388971 10 gce_loadbalancer_external.go:267] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Creating forwarding rule, IP 35.212.229.111 (tier: Standard).
I0607 16:33:49.587138 10 gen.go:9055] GCEForwardingRules.Insert(context.Background.WithDeadline(2021-06-07 17:33:34.388994428 +0000 UTC m=+4793.121092228 [59m44.80184659s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}, &{IPAddress:35.212.229.111 IPProtocol:TCP AllPorts:false AllowGlobalAccess:false BackendService: CreationTimestamp: Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Fingerprint: Id:0 IpVersion: IsMirroringCollector:false Kind: LoadBalancingScheme: MetadataFilters:[] Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PortRange:80-80 Ports:[] Region: SelfLink: ServiceLabel: ServiceName: Subnetwork: Target:https://compute.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3 ServerResponse:{HTTPStatusCode:0 Header:map[]} ForceSendFields:[] NullFields:[]}) = <nil>
I0607 16:33:49.587218 10 gce_loadbalancer_external.go:276] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Created forwarding rule, IP 35.212.229.111.
I0607 16:33:51.438642 10 gen.go:2776] GCEAddresses.Delete(context.Background.WithDeadline(2021-06-07 17:33:49.587226016 +0000 UTC m=+4808.319323818 [59m58.148561388s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = <nil>
I0607 16:33:51.438720 10 gce_loadbalancer_external.go:136] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Released static IP 35.212.229.111.
I0607 16:33:51.438732 10 gce_loadbalancer.go:177] EnsureLoadBalancer(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1): done ensuring loadbalancer.
I0607 16:34:24.835804 10 gce_loadbalancer.go:137] EnsureLoadBalancer(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1): ensure EXTERNAL loadbalancer
I0607 16:34:25.032546 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:34:24.835811178 +0000 UTC m=+4843.567908981 [59m59.803240454s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = &{IPAddress:35.212.229.111 IPProtocol:TCP AllPorts:false AllowGlobalAccess:false BackendService: CreationTimestamp:2021-06-07T09:33:35.004-07:00 Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Fingerprint:VVaO89EnduI= Id:3750318871430821040 IpVersion: IsMirroringCollector:false Kind:compute#forwardingRule LoadBalancingScheme:EXTERNAL MetadataFilters:[] Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PortRange:80-80 Ports:[] Region:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1 SelfLink:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3 ServiceLabel: ServiceName: Subnetwork: Target:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3 ServerResponse:{HTTPStatusCode:200 Header:map[Cache-Control:[private] Content-Type:[application/json; charset=UTF-8] Date:[Mon, 07 Jun 2021 16:34:25 GMT] Etag:[BzAeD0mKd5P8A3wZn6RNN9pyMls=/PQb0LHX2n4BU0aRH8MsgvptFhVw=] Server:[ESF] Vary:[Origin X-Origin Referer] X-Content-Type-Options:[nosniff] X-Frame-Options:[SAMEORIGIN] X-Xss-Protection:[0]]} ForceSendFields:[] NullFields:[]}, <nil>
I0607 16:34:25.183734 10 gce_loadbalancer_external.go:74] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc), us-west1, , [TCP/80], [bootstrap-e2e-minion-group-gp1b bootstrap-e2e-minion-group-rrsg bootstrap-e2e-master bootstrap-e2e-minion-group-5p5s], map[])
I0607 16:34:25.183750 10 gce_loadbalancer_external.go:83] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Desired network tier "Premium".
I0607 16:34:25.373932 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:34:25.183758644 +0000 UTC m=+4843.915856452 [59m59.809803679s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = &{IPAddress:35.212.229.111 IPProtocol:TCP AllPorts:false AllowGlobalAccess:false BackendService: CreationTimestamp:2021-06-07T09:33:35.004-07:00 Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Fingerprint:VVaO89EnduI= Id:3750318871430821040 IpVersion: IsMirroringCollector:false Kind:compute#forwardingRule LoadBalancingScheme:EXTERNAL MetadataFilters:[] Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PortRange:80-80 Ports:[] Region:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1 SelfLink:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3 ServiceLabel: ServiceName: Subnetwork: Target:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3 ServerResponse:{HTTPStatusCode:200 Header:map[Cache-Control:[private] Content-Type:[application/json; charset=UTF-8] Date:[Mon, 07 Jun 2021 16:34:25 GMT] Etag:[-J4M4IbntmC3wJ_v22j-MWeujcA=/yDpWUgcOY3MnDekLcqWsbCXqRM4=] Server:[ESF] Vary:[Origin X-Origin Referer] X-Content-Type-Options:[nosniff] X-Frame-Options:[SAMEORIGIN] X-Xss-Protection:[0]]} ForceSendFields:[] NullFields:[]}, <nil>
I0607 16:34:25.660665 10 gen.go:2738] GCEAddresses.Insert(context.Background.WithDeadline(2021-06-07 17:34:25.374068424 +0000 UTC m=+4844.106166217 [59m59.713395731s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}, ...) = googleapi: Error 400: The network tier of external IP is STANDARD, that of Address must be the same., badRequest
W0607 16:34:25.784797 10 gce_loadbalancer_external.go:139] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Orphaning static IP in region us-west1: <nil>.
E0607 16:34:25.784833 10 gce_loadbalancer.go:174] Failed to EnsureLoadBalancer(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1), err: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP "35.212.229.111" was not found in region "us-west1"
E0607 16:34:25.784868 10 controller.go:307] error processing service services-6097/net-tiers-svc (will retry): failed to ensure load balancer: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP "35.212.229.111" was not found in region "us-west1"
I0607 16:34:25.785125 10 event.go:291] "Event occurred" object="services-6097/net-tiers-svc" kind="Service" apiVersion="v1" type="Warning" reason="SyncLoadBalancerFailed" message="Error syncing load balancer: failed to ensure load balancer: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP \"35.212.229.111\" was not found in region \"us-west1\""
I0607 16:34:30.785840 10 gce_loadbalancer.go:137] EnsureLoadBalancer(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1): ensure EXTERNAL loadbalancer
I0607 16:34:30.965929 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:34:30.785848855 +0000 UTC m=+4849.517946661 [59m59.819908357s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = &{IPAddress:35.212.229.111 IPProtocol:TCP AllPorts:false AllowGlobalAccess:false BackendService: CreationTimestamp:2021-06-07T09:33:35.004-07:00 Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Fingerprint:VVaO89EnduI= Id:3750318871430821040 IpVersion: IsMirroringCollector:false Kind:compute#forwardingRule LoadBalancingScheme:EXTERNAL MetadataFilters:[] Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PortRange:80-80 Ports:[] Region:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1 SelfLink:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3 ServiceLabel: ServiceName: Subnetwork: Target:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3 ServerResponse:{HTTPStatusCode:200 Header:map[Cache-Control:[private] Content-Type:[application/json; charset=UTF-8] Date:[Mon, 07 Jun 2021 16:34:30 GMT] Etag:[Lg-DboxBO7_kdNk3-qG_yaRatPw=/fYG0LpIXgKMon4nsDJAxhOlk8jE=] Server:[ESF] Vary:[Origin X-Origin Referer] X-Content-Type-Options:[nosniff] X-Frame-Options:[SAMEORIGIN] X-Xss-Protection:[0]]} ForceSendFields:[] NullFields:[]}, <nil>
I0607 16:34:31.131210 10 gce_loadbalancer_external.go:74] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc), us-west1, , [TCP/80], [bootstrap-e2e-minion-group-gp1b bootstrap-e2e-minion-group-rrsg bootstrap-e2e-master bootstrap-e2e-minion-group-5p5s], map[])
I0607 16:34:31.131226 10 gce_loadbalancer_external.go:83] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Desired network tier "Premium".
I0607 16:34:31.314990 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:34:31.131234199 +0000 UTC m=+4849.863332002 [59m59.816233247s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = &{IPAddress:35.212.229.111 IPProtocol:TCP AllPorts:false AllowGlobalAccess:false BackendService: CreationTimestamp:2021-06-07T09:33:35.004-07:00 Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Fingerprint:VVaO89EnduI= Id:3750318871430821040 IpVersion: IsMirroringCollector:false Kind:compute#forwardingRule LoadBalancingScheme:EXTERNAL MetadataFilters:[] Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PortRange:80-80 Ports:[] Region:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1 SelfLink:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3 ServiceLabel: ServiceName: Subnetwork: Target:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3 ServerResponse:{HTTPStatusCode:200 Header:map[Cache-Control:[private] Content-Type:[application/json; charset=UTF-8] Date:[Mon, 07 Jun 2021 16:34:31 GMT] Etag:[nFcxFoo0bZDYkfqlZwmhNv2z5fQ=/DhVQMHyagmHBZqaq0CDAfWba9jU=] Server:[ESF] Vary:[Origin X-Origin Referer] X-Content-Type-Options:[nosniff] X-Frame-Options:[SAMEORIGIN] X-Xss-Protection:[0]]} ForceSendFields:[] NullFields:[]}, <nil>
I0607 16:34:31.727919 10 gen.go:2738] GCEAddresses.Insert(context.Background.WithDeadline(2021-06-07 17:34:31.315091833 +0000 UTC m=+4850.047189631 [59m59.58716092s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}, ...) = googleapi: Error 400: The network tier of external IP is STANDARD, that of Address must be the same., badRequest
W0607 16:34:31.859096 10 gce_loadbalancer_external.go:139] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Orphaning static IP in region us-west1: <nil>.
E0607 16:34:31.859107 10 gce_loadbalancer.go:174] Failed to EnsureLoadBalancer(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1), err: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP "35.212.229.111" was not found in region "us-west1"
E0607 16:34:31.859154 10 controller.go:307] error processing service services-6097/net-tiers-svc (will retry): failed to ensure load balancer: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP "35.212.229.111" was not found in region "us-west1"
I0607 16:34:31.859903 10 event.go:291] "Event occurred" object="services-6097/net-tiers-svc" kind="Service" apiVersion="v1" type="Warning" reason="SyncLoadBalancerFailed" message="Error syncing load balancer: failed to ensure load balancer: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP \"35.212.229.111\" was not found in region \"us-west1\""
I0607 16:34:41.860919 10 gce_loadbalancer.go:137] EnsureLoadBalancer(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1): ensure EXTERNAL loadbalancer
I0607 16:34:42.031771 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:34:41.860962382 +0000 UTC m=+4860.593060189 [59m59.829181036s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = &{IPAddress:35.212.229.111 IPProtocol:TCP AllPorts:false AllowGlobalAccess:false BackendService: CreationTimestamp:2021-06-07T09:33:35.004-07:00 Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Fingerprint:VVaO89EnduI= Id:3750318871430821040 IpVersion: IsMirroringCollector:false Kind:compute#forwardingRule LoadBalancingScheme:EXTERNAL MetadataFilters:[] Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PortRange:80-80 Ports:[] Region:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1 SelfLink:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3 ServiceLabel: ServiceName: Subnetwork: Target:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3 ServerResponse:{HTTPStatusCode:200 Header:map[Cache-Control:[private] Content-Type:[application/json; charset=UTF-8] Date:[Mon, 07 Jun 2021 16:34:42 GMT] Etag:[MlIe0Ilkkamw3wBYDZN_hbOaQSw=/moyK2X7rOZIB_zAa2yifJC4397Q=] Server:[ESF] Vary:[Origin X-Origin Referer] X-Content-Type-Options:[nosniff] X-Frame-Options:[SAMEORIGIN] X-Xss-Protection:[0]]} ForceSendFields:[] NullFields:[]}, <nil>
I0607 16:34:42.191211 10 gce_loadbalancer_external.go:74] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc), us-west1, , [TCP/80], [bootstrap-e2e-master bootstrap-e2e-minion-group-5p5s bootstrap-e2e-minion-group-gp1b bootstrap-e2e-minion-group-rrsg], map[])
I0607 16:34:42.191227 10 gce_loadbalancer_external.go:83] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Desired network tier "Premium".
I0607 16:34:42.381511 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:34:42.191235269 +0000 UTC m=+4860.923333074 [59m59.809713611s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = &{IPAddress:35.212.229.111 IPProtocol:TCP AllPorts:false AllowGlobalAccess:false BackendService: CreationTimestamp:2021-06-07T09:33:35.004-07:00 Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Fingerprint:VVaO89EnduI= Id:3750318871430821040 IpVersion: IsMirroringCollector:false Kind:compute#forwardingRule LoadBalancingScheme:EXTERNAL MetadataFilters:[] Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PortRange:80-80 Ports:[] Region:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1 SelfLink:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3 ServiceLabel: ServiceName: Subnetwork: Target:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3 ServerResponse:{HTTPStatusCode:200 Header:map[Cache-Control:[private] Content-Type:[application/json; charset=UTF-8] Date:[Mon, 07 Jun 2021 16:34:42 GMT] Etag:[QKqO2baIgFiJOCbscU7sB46LG6M=/YZIjOwrGyKYaGvOa7700JyypZA8=] Server:[ESF] Vary:[Origin X-Origin Referer] X-Content-Type-Options:[nosniff] X-Frame-Options:[SAMEORIGIN] X-Xss-Protection:[0]]} ForceSendFields:[] NullFields:[]}, <nil>
I0607 16:34:42.810716 10 gen.go:2738] GCEAddresses.Insert(context.Background.WithDeadline(2021-06-07 17:34:42.381626833 +0000 UTC m=+4861.113724629 [59m59.570902883s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}, ...) = googleapi: Error 400: The network tier of external IP is STANDARD, that of Address must be the same., badRequest
W0607 16:34:42.934230 10 gce_loadbalancer_external.go:139] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Orphaning static IP in region us-west1: <nil>.
E0607 16:34:42.934246 10 gce_loadbalancer.go:174] Failed to EnsureLoadBalancer(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1), err: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP "35.212.229.111" was not found in region "us-west1"
E0607 16:34:42.934297 10 controller.go:307] error processing service services-6097/net-tiers-svc (will retry): failed to ensure load balancer: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP "35.212.229.111" was not found in region "us-west1"
I0607 16:34:42.934663 10 event.go:291] "Event occurred" object="services-6097/net-tiers-svc" kind="Service" apiVersion="v1" type="Warning" reason="SyncLoadBalancerFailed" message="Error syncing load balancer: failed to ensure load balancer: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP \"35.212.229.111\" was not found in region \"us-west1\""
I0607 16:35:02.935865 10 gce_loadbalancer.go:137] EnsureLoadBalancer(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1): ensure EXTERNAL loadbalancer
I0607 16:35:03.132420 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:35:02.935904008 +0000 UTC m=+4881.668001814 [59m59.803468253s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = &{IPAddress:35.212.229.111 IPProtocol:TCP AllPorts:false AllowGlobalAccess:false BackendService: CreationTimestamp:2021-06-07T09:33:35.004-07:00 Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Fingerprint:VVaO89EnduI= Id:3750318871430821040 IpVersion: IsMirroringCollector:false Kind:compute#forwardingRule LoadBalancingScheme:EXTERNAL MetadataFilters:[] Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PortRange:80-80 Ports:[] Region:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1 SelfLink:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3 ServiceLabel: ServiceName: Subnetwork: Target:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3 ServerResponse:{HTTPStatusCode:200 Header:map[Cache-Control:[private] Content-Type:[application/json; charset=UTF-8] Date:[Mon, 07 Jun 2021 16:35:03 GMT] Etag:[d-R8HeMPVeYuCVGsSfusC6GDDKk=/RrKq73OU3Fzu7g_JtkE4FTJ2eZQ=] Server:[ESF] Vary:[Origin X-Origin Referer] X-Content-Type-Options:[nosniff] X-Frame-Options:[SAMEORIGIN] X-Xss-Protection:[0]]} ForceSendFields:[] NullFields:[]}, <nil>
I0607 16:35:03.293974 10 gce_loadbalancer_external.go:74] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc), us-west1, , [TCP/80], [bootstrap-e2e-minion-group-rrsg bootstrap-e2e-master bootstrap-e2e-minion-group-5p5s bootstrap-e2e-minion-group-gp1b], map[])
I0607 16:35:03.293994 10 gce_loadbalancer_external.go:83] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Desired network tier "Premium".
I0607 16:35:03.489133 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:35:03.294002075 +0000 UTC m=+4882.026099882 [59m59.804858228s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = &{IPAddress:35.212.229.111 IPProtocol:TCP AllPorts:false AllowGlobalAccess:false BackendService: CreationTimestamp:2021-06-07T09:33:35.004-07:00 Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Fingerprint:VVaO89EnduI= Id:3750318871430821040 IpVersion: IsMirroringCollector:false Kind:compute#forwardingRule LoadBalancingScheme:EXTERNAL MetadataFilters:[] Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PortRange:80-80 Ports:[] Region:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1 SelfLink:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3 ServiceLabel: ServiceName: Subnetwork: Target:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3 ServerResponse:{HTTPStatusCode:200 Header:map[Cache-Control:[private] Content-Type:[application/json; charset=UTF-8] Date:[Mon, 07 Jun 2021 16:35:03 GMT] Etag:[d-R8HeMPVeYuCVGsSfusC6GDDKk=/RrKq73OU3Fzu7g_JtkE4FTJ2eZQ=] Server:[ESF] Vary:[Origin X-Origin Referer] X-Content-Type-Options:[nosniff] X-Frame-Options:[SAMEORIGIN] X-Xss-Protection:[0]]} ForceSendFields:[] NullFields:[]}, <nil>
I0607 16:35:03.907270 10 gen.go:2738] GCEAddresses.Insert(context.Background.WithDeadline(2021-06-07 17:35:03.489223401 +0000 UTC m=+4882.221321200 [59m59.58194264s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}, ...) = googleapi: Error 400: The network tier of external IP is STANDARD, that of Address must be the same., badRequest
W0607 16:35:04.021522 10 gce_loadbalancer_external.go:139] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Orphaning static IP in region us-west1: <nil>.
E0607 16:35:04.021560 10 gce_loadbalancer.go:174] Failed to EnsureLoadBalancer(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1), err: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP "35.212.229.111" was not found in region "us-west1"
E0607 16:35:04.021594 10 controller.go:307] error processing service services-6097/net-tiers-svc (will retry): failed to ensure load balancer: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP "35.212.229.111" was not found in region "us-west1"
I0607 16:35:04.021958 10 event.go:291] "Event occurred" object="services-6097/net-tiers-svc" kind="Service" apiVersion="v1" type="Warning" reason="SyncLoadBalancerFailed" message="Error syncing load balancer: failed to ensure load balancer: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP \"35.212.229.111\" was not found in region \"us-west1\""
I0607 16:35:44.021776 10 gce_loadbalancer.go:137] EnsureLoadBalancer(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1): ensure EXTERNAL loadbalancer
I0607 16:35:44.210286 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:35:44.021807969 +0000 UTC m=+4922.753905777 [59m59.811509908s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = &{IPAddress:35.212.229.111 IPProtocol:TCP AllPorts:false AllowGlobalAccess:false BackendService: CreationTimestamp:2021-06-07T09:33:35.004-07:00 Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Fingerprint:VVaO89EnduI= Id:3750318871430821040 IpVersion: IsMirroringCollector:false Kind:compute#forwardingRule LoadBalancingScheme:EXTERNAL MetadataFilters:[] Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PortRange:80-80 Ports:[] Region:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1 SelfLink:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3 ServiceLabel: ServiceName: Subnetwork: Target:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3 ServerResponse:{HTTPStatusCode:200 Header:map[Cache-Control:[private] Content-Type:[application/json; charset=UTF-8] Date:[Mon, 07 Jun 2021 16:35:44 GMT] Etag:[DB2PTr1wrjJCI8lSvdnzYxloF9Y=/EKQJ0iN37qvMvWWvGiDI4wkj0N4=] Server:[ESF] Vary:[Origin X-Origin Referer] X-Content-Type-Options:[nosniff] X-Frame-Options:[SAMEORIGIN] X-Xss-Protection:[0]]} ForceSendFields:[] NullFields:[]}, <nil>
I0607 16:35:44.361176 10 gce_loadbalancer_external.go:74] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc), us-west1, , [TCP/80], [bootstrap-e2e-master bootstrap-e2e-minion-group-5p5s bootstrap-e2e-minion-group-gp1b bootstrap-e2e-minion-group-rrsg], map[])
I0607 16:35:44.361207 10 gce_loadbalancer_external.go:83] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Desired network tier "Premium".
I0607 16:35:44.563572 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:35:44.361214888 +0000 UTC m=+4923.093312694 [59m59.797632027s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = &{IPAddress:35.212.229.111 IPProtocol:TCP AllPorts:false AllowGlobalAccess:false BackendService: CreationTimestamp:2021-06-07T09:33:35.004-07:00 Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Fingerprint:VVaO89EnduI= Id:3750318871430821040 IpVersion: IsMirroringCollector:false Kind:compute#forwardingRule LoadBalancingScheme:EXTERNAL MetadataFilters:[] Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PortRange:80-80 Ports:[] Region:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1 SelfLink:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3 ServiceLabel: ServiceName: Subnetwork: Target:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3 ServerResponse:{HTTPStatusCode:200 Header:map[Cache-Control:[private] Content-Type:[application/json; charset=UTF-8] Date:[Mon, 07 Jun 2021 16:35:44 GMT] Etag:[cvIFaR4xMeZ0XNeXWCkLjr-Jvzc=/eNZ15hQkMdM-Kis5NqxGzcZh1HQ=] Server:[ESF] Vary:[Origin X-Origin Referer] X-Content-Type-Options:[nosniff] X-Frame-Options:[SAMEORIGIN] X-Xss-Protection:[0]]} ForceSendFields:[] NullFields:[]}, <nil>
I0607 16:35:44.986190 10 gen.go:2738] GCEAddresses.Insert(context.Background.WithDeadline(2021-06-07 17:35:44.563664067 +0000 UTC m=+4923.295761866 [59m59.57746291s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}, ...) = googleapi: Error 400: The network tier of external IP is STANDARD, that of Address must be the same., badRequest
W0607 16:35:45.118747 10 gce_loadbalancer_external.go:139] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Orphaning static IP in region us-west1: <nil>.
E0607 16:35:45.118759 10 gce_loadbalancer.go:174] Failed to EnsureLoadBalancer(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1), err: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP "35.212.229.111" was not found in region "us-west1"
E0607 16:35:45.118808 10 controller.go:307] error processing service services-6097/net-tiers-svc (will retry): failed to ensure load balancer: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP "35.212.229.111" was not found in region "us-west1"
I0607 16:35:45.119317 10 event.go:291] "Event occurred" object="services-6097/net-tiers-svc" kind="Service" apiVersion="v1" type="Warning" reason="SyncLoadBalancerFailed" message="Error syncing load balancer: failed to ensure load balancer: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP \"35.212.229.111\" was not found in region \"us-west1\""
I0607 16:37:05.088913 10 gen.go:13598] GCEHttpHealthChecks.Delete(context.Background.WithDeadline(2021-06-07 17:37:03.529039479 +0000 UTC m=+5002.261137280 [59m58.440118836s]), Key{"k8s-fde405acdccef8bb-node"}) = googleapi: Error 400: RESOURCE_IN_USE_BY_ANOTHER_RESOURCE - The http_health_check resource 'projects/k8s-jkns-gci-gce-ingress-1-6/global/httpHealthChecks/k8s-fde405acdccef8bb-node' is already being used by 'projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3'
I0607 16:37:05.088974 10 gce_loadbalancer_external.go:388] DeleteExternalTargetPoolAndChecks(a123acb7c80eb4a7a924d50192b2aaa3(esipp-1773/external-local-nodes)): Health check k8s-fde405acdccef8bb-node is in used: googleapi: Error 400: RESOURCE_IN_USE_BY_ANOTHER_RESOURCE - The http_health_check resource 'projects/k8s-jkns-gci-gce-ingress-1-6/global/httpHealthChecks/k8s-fde405acdccef8bb-node' is already being used by 'projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3'.
I0607 16:37:05.411366 10 gce_loadbalancer.go:137] EnsureLoadBalancer(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1): ensure EXTERNAL loadbalancer
I0607 16:37:05.588753 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:37:05.41137352 +0000 UTC m=+5004.143471324 [59m59.822609051s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = &{IPAddress:35.212.229.111 IPProtocol:TCP AllPorts:false AllowGlobalAccess:false BackendService: CreationTimestamp:2021-06-07T09:33:35.004-07:00 Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Fingerprint:VVaO89EnduI= Id:3750318871430821040 IpVersion: IsMirroringCollector:false Kind:compute#forwardingRule LoadBalancingScheme:EXTERNAL MetadataFilters:[] Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PortRange:80-80 Ports:[] Region:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1 SelfLink:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3 ServiceLabel: ServiceName: Subnetwork: Target:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3 ServerResponse:{HTTPStatusCode:200 Header:map[Cache-Control:[private] Content-Type:[application/json; charset=UTF-8] Date:[Mon, 07 Jun 2021 16:37:05 GMT] Etag:[6p8yDS3lVRHdnk4YncQ_twbV80A=/ZDok-MPliMfGpmS6i3Nu4H8qNQY=] Server:[ESF] Vary:[Origin X-Origin Referer] X-Content-Type-Options:[nosniff] X-Frame-Options:[SAMEORIGIN] X-Xss-Protection:[0]]} ForceSendFields:[] NullFields:[]}, <nil>
I0607 16:37:05.759590 10 gce_loadbalancer_external.go:74] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc), us-west1, , [TCP/80], [bootstrap-e2e-master bootstrap-e2e-minion-group-5p5s bootstrap-e2e-minion-group-gp1b bootstrap-e2e-minion-group-rrsg], map[])
I0607 16:37:05.759607 10 gce_loadbalancer_external.go:83] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Desired network tier "Premium".
I0607 16:37:05.934000 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:37:05.759615262 +0000 UTC m=+5004.491713068 [59m59.82560444s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = &{IPAddress:35.212.229.111 IPProtocol:TCP AllPorts:false AllowGlobalAccess:false BackendService: CreationTimestamp:2021-06-07T09:33:35.004-07:00 Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Fingerprint:VVaO89EnduI= Id:3750318871430821040 IpVersion: IsMirroringCollector:false Kind:compute#forwardingRule LoadBalancingScheme:EXTERNAL MetadataFilters:[] Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PortRange:80-80 Ports:[] Region:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1 SelfLink:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3 ServiceLabel: ServiceName: Subnetwork: Target:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3 ServerResponse:{HTTPStatusCode:200 Header:map[Cache-Control:[private] Content-Type:[application/json; charset=UTF-8] Date:[Mon, 07 Jun 2021 16:37:05 GMT] Etag:[W4dCyNsM8_o-uDP8bznSk7pQhDc=/1pefNFSY91yluFb4nQPrUtsGRrM=] Server:[ESF] Vary:[Origin X-Origin Referer] X-Content-Type-Options:[nosniff] X-Frame-Options:[SAMEORIGIN] X-Xss-Protection:[0]]} ForceSendFields:[] NullFields:[]}, <nil>
I0607 16:37:06.333149 10 gen.go:2738] GCEAddresses.Insert(context.Background.WithDeadline(2021-06-07 17:37:05.934106356 +0000 UTC m=+5004.666204155 [59m59.600945227s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}, ...) = googleapi: Error 400: The network tier of external IP is STANDARD, that of Address must be the same., badRequest
W0607 16:37:06.467058 10 gce_loadbalancer_external.go:139] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Orphaning static IP in region us-west1: <nil>.
E0607 16:37:06.467103 10 gce_loadbalancer.go:174] Failed to EnsureLoadBalancer(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1), err: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP "35.212.229.111" was not found in region "us-west1"
E0607 16:37:06.467144 10 controller.go:307] error processing service services-6097/net-tiers-svc (will retry): failed to ensure load balancer: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP "35.212.229.111" was not found in region "us-west1"
I0607 16:37:06.467777 10 event.go:291] "Event occurred" object="services-6097/net-tiers-svc" kind="Service" apiVersion="v1" type="Warning" reason="SyncLoadBalancerFailed" message="Error syncing load balancer: failed to ensure load balancer: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP \"35.212.229.111\" was not found in region \"us-west1\""
I0607 16:39:46.468299 10 gce_loadbalancer.go:137] EnsureLoadBalancer(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1): ensure EXTERNAL loadbalancer
I0607 16:39:46.637976 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:39:46.468331858 +0000 UTC m=+5165.200429666 [59m59.830344034s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = &{IPAddress:35.212.229.111 IPProtocol:TCP AllPorts:false AllowGlobalAccess:false BackendService: CreationTimestamp:2021-06-07T09:33:35.004-07:00 Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Fingerprint:VVaO89EnduI= Id:3750318871430821040 IpVersion: IsMirroringCollector:false Kind:compute#forwardingRule LoadBalancingScheme:EXTERNAL MetadataFilters:[] Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PortRange:80-80 Ports:[] Region:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1 SelfLink:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3 ServiceLabel: ServiceName: Subnetwork: Target:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3 ServerResponse:{HTTPStatusCode:200 Header:map[Cache-Control:[private] Content-Type:[application/json; charset=UTF-8] Date:[Mon, 07 Jun 2021 16:39:46 GMT] Etag:[qCwr9gfN4yGLp7gKDR1QK0q-N9o=/vyX7HC_smY6da9MIc8TGZ5o6KQM=] Server:[ESF] Vary:[Origin X-Origin Referer] X-Content-Type-Options:[nosniff] X-Frame-Options:[SAMEORIGIN] X-Xss-Protection:[0]]} ForceSendFields:[] NullFields:[]}, <nil>
I0607 16:39:46.916984 10 gce_loadbalancer_external.go:74] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc), us-west1, , [TCP/80], [bootstrap-e2e-minion-group-5p5s bootstrap-e2e-minion-group-gp1b bootstrap-e2e-minion-group-rrsg bootstrap-e2e-master], map[])
I0607 16:39:46.917009 10 gce_loadbalancer_external.go:83] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Desired network tier "Premium".
I0607 16:39:47.089342 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:39:46.917016868 +0000 UTC m=+5165.649114674 [59m59.827663442s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = &{IPAddress:35.212.229.111 IPProtocol:TCP AllPorts:false AllowGlobalAccess:false BackendService: CreationTimestamp:2021-06-07T09:33:35.004-07:00 Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Fingerprint:VVaO89EnduI= Id:3750318871430821040 IpVersion: IsMirroringCollector:false Kind:compute#forwardingRule LoadBalancingScheme:EXTERNAL MetadataFilters:[] Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PortRange:80-80 Ports:[] Region:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1 SelfLink:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3 ServiceLabel: ServiceName: Subnetwork: Target:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3 ServerResponse:{HTTPStatusCode:200 Header:map[Cache-Control:[private] Content-Type:[application/json; charset=UTF-8] Date:[Mon, 07 Jun 2021 16:39:47 GMT] Etag:[7pQmt6uva5lTWbuvEsPTEq5mTxI=/fGOmTKhKBBpeWJfSdw4j9AcqP7s=] Server:[ESF] Vary:[Origin X-Origin Referer] X-Content-Type-Options:[nosniff] X-Frame-Options:[SAMEORIGIN] X-Xss-Protection:[0]]} ForceSendFields:[] NullFields:[]}, <nil>
I0607 16:39:47.519038 10 gen.go:2738] GCEAddresses.Insert(context.Background.WithDeadline(2021-06-07 17:39:47.089424673 +0000 UTC m=+5165.821522474 [59m59.570376756s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}, ...) = googleapi: Error 400: The network tier of external IP is STANDARD, that of Address must be the same., badRequest
W0607 16:39:47.664022 10 gce_loadbalancer_external.go:139] ensureExternalLoadBalancer(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Orphaning static IP in region us-west1: <nil>.
E0607 16:39:47.664044 10 gce_loadbalancer.go:174] Failed to EnsureLoadBalancer(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1), err: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP "35.212.229.111" was not found in region "us-west1"
E0607 16:39:47.664088 10 controller.go:307] error processing service services-6097/net-tiers-svc (will retry): failed to ensure load balancer: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP "35.212.229.111" was not found in region "us-west1"
I0607 16:39:47.664490 10 event.go:291] "Event occurred" object="services-6097/net-tiers-svc" kind="Service" apiVersion="v1" type="Warning" reason="SyncLoadBalancerFailed" message="Error syncing load balancer: failed to ensure load balancer: failed to ensure a static IP for load balancer (a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): error getting static IP address: googleapi: Error 404: Address with IP \"35.212.229.111\" was not found in region \"us-west1\""
I0607 16:44:29.900313 10 gen.go:13598] GCEHttpHealthChecks.Delete(context.Background.WithDeadline(2021-06-07 17:44:28.28449138 +0000 UTC m=+5447.016589182 [59m58.384168388s]), Key{"k8s-fde405acdccef8bb-node"}) = googleapi: Error 400: RESOURCE_IN_USE_BY_ANOTHER_RESOURCE - The http_health_check resource 'projects/k8s-jkns-gci-gce-ingress-1-6/global/httpHealthChecks/k8s-fde405acdccef8bb-node' is already being used by 'projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3'
I0607 16:44:29.900380 10 gce_loadbalancer_external.go:388] DeleteExternalTargetPoolAndChecks(a50b1324fbc714043b7df95173b9a59d(loadbalancers-8490/lb-finalizer)): Health check k8s-fde405acdccef8bb-node is in used: googleapi: Error 400: RESOURCE_IN_USE_BY_ANOTHER_RESOURCE - The http_health_check resource 'projects/k8s-jkns-gci-gce-ingress-1-6/global/httpHealthChecks/k8s-fde405acdccef8bb-node' is already being used by 'projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3'.
I0607 16:44:30.103143 10 gen.go:8979] GCEForwardingRules.Get(context.Background.WithDeadline(2021-06-07 17:44:29.924138678 +0000 UTC m=+5448.656236466 [59m59.820984734s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = &{IPAddress:35.212.229.111 IPProtocol:TCP AllPorts:false AllowGlobalAccess:false BackendService: CreationTimestamp:2021-06-07T09:33:35.004-07:00 Description:{"kubernetes.io/service-name":"services-6097/net-tiers-svc"} Fingerprint:VVaO89EnduI= Id:3750318871430821040 IpVersion: IsMirroringCollector:false Kind:compute#forwardingRule LoadBalancingScheme:EXTERNAL MetadataFilters:[] Name:a8d573ae414fd4ec099dff43893319d3 Network: NetworkTier:STANDARD PortRange:80-80 Ports:[] Region:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1 SelfLink:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/forwardingRules/a8d573ae414fd4ec099dff43893319d3 ServiceLabel: ServiceName: Subnetwork: Target:https://www.googleapis.com/compute/v1/projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/targetPools/a8d573ae414fd4ec099dff43893319d3 ServerResponse:{HTTPStatusCode:200 Header:map[Cache-Control:[private] Content-Type:[application/json; charset=UTF-8] Date:[Mon, 07 Jun 2021 16:44:30 GMT] Etag:[iLmu-AEmw-a389fyH0TtgVfcshA=/wyuhS7ePIMR8MvVKj5QiM2DTlxw=] Server:[ESF] Vary:[Origin X-Origin Referer] X-Content-Type-Options:[nosniff] X-Frame-Options:[SAMEORIGIN] X-Xss-Protection:[0]]} ForceSendFields:[] NullFields:[]}, <nil>
I0607 16:44:30.103326 10 gce_loadbalancer.go:211] EnsureLoadBalancerDeleted(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1): deleting loadbalancer
I0607 16:44:30.103367 10 gce_loadbalancer_external.go:342] ensureExternalLoadBalancerDeleted(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Deleting forwarding rule.
I0607 16:44:30.104067 10 gce_loadbalancer_external.go:324] ensureExternalLoadBalancerDeleted(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Deleting firewall rule.
I0607 16:44:30.104560 10 gce_loadbalancer_external.go:338] ensureExternalLoadBalancerDeleted(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Deleting IP address.
I0607 16:44:30.229995 10 gen.go:2771] GCEAddresses.Delete(context.Background.WithDeadline(2021-06-07 17:44:30.104667276 +0000 UTC m=+5448.836765079 [59m59.874660713s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = googleapi: Error 404: The resource 'projects/k8s-jkns-gci-gce-ingress-1-6/regions/us-west1/addresses/a8d573ae414fd4ec099dff43893319d3' was not found, notFound
I0607 16:44:36.660075 10 gen.go:8711] GCEFirewalls.Delete(context.Background.WithDeadline(2021-06-07 17:44:30.104073642 +0000 UTC m=+5448.836171444 [59m53.443986874s]), Key{"k8s-fw-a8d573ae414fd4ec099dff43893319d3"}) = <nil>
I0607 16:44:46.158953 10 gen.go:9088] GCEForwardingRules.Delete(context.Background.WithDeadline(2021-06-07 17:44:30.103372003 +0000 UTC m=+5448.835469800 [59m43.944407705s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = <nil>
I0607 16:44:46.159019 10 gce_loadbalancer_external.go:348] ensureExternalLoadBalancerDeleted(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Deleting target pool.
I0607 16:44:50.248920 10 gen.go:28163] GCETargetPools.Delete(context.Background.WithDeadline(2021-06-07 17:44:46.159029096 +0000 UTC m=+5464.891126907 [59m55.910090868s]), Key{"a8d573ae414fd4ec099dff43893319d3", region: "us-west1"}) = <nil>
I0607 16:44:50.248972 10 gce_loadbalancer_external.go:384] DeleteExternalTargetPoolAndChecks(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Deleting health check a8d573ae414fd4ec099dff43893319d3.
I0607 16:44:51.015334 10 gen.go:13593] GCEHttpHealthChecks.Delete(context.Background.WithDeadline(2021-06-07 17:44:50.24897735 +0000 UTC m=+5468.981075151 [59m59.233633762s]), Key{"a8d573ae414fd4ec099dff43893319d3"}) = googleapi: Error 404: The resource 'projects/k8s-jkns-gci-gce-ingress-1-6/global/httpHealthChecks/a8d573ae414fd4ec099dff43893319d3' was not found, notFound
I0607 16:44:51.015387 10 gce_loadbalancer_external.go:401] DeleteExternalTargetPoolAndChecks(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Health check a8d573ae414fd4ec099dff43893319d3 is already deleted.
I0607 16:44:51.015396 10 gce_loadbalancer_external.go:406] DeleteExternalTargetPoolAndChecks(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Deleting health check firewall k8s-a8d573ae414fd4ec099dff43893319d3-http-hc.
I0607 16:44:51.119773 10 gen.go:8706] GCEFirewalls.Delete(context.Background.WithDeadline(2021-06-07 17:44:51.015412282 +0000 UTC m=+5469.747510082 [59m59.895617372s]), Key{"k8s-a8d573ae414fd4ec099dff43893319d3-http-hc"}) = googleapi: Error 404: The resource 'projects/k8s-jkns-gci-gce-ingress-1-6/global/firewalls/k8s-a8d573ae414fd4ec099dff43893319d3-http-hc' was not found, notFound
I0607 16:44:51.119843 10 gce_loadbalancer_external.go:384] DeleteExternalTargetPoolAndChecks(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Deleting health check k8s-fde405acdccef8bb-node.
I0607 16:44:54.363502 10 gce_loadbalancer_external.go:406] DeleteExternalTargetPoolAndChecks(a8d573ae414fd4ec099dff43893319d3(services-6097/net-tiers-svc)): Deleting health check firewall k8s-fde405acdccef8bb-node-http-hc.
I0607 16:45:01.382746 10 gce_loadbalancer.go:219] EnsureLoadBalancerDeleted(bootstrap-e2e, services-6097, net-tiers-svc, a8d573ae414fd4ec099dff43893319d3, us-west1): done deleting loadbalancer. err: <nil>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment