Ingress failing on transit-radar

.sh

# port forwarding works
% kubectl port-forward svc/transit-radar 8080:80
Forwarding from 127.0.0.1:8080 -> 80
Forwarding from [::1]:8080 -> 80
Handling connection for 8080

# but using the ingress fails with 502, the ingress controller logs show it fails to connect to 192.168.99.20:80 (the pod IP)
% kubectl logs nginx-ingress-controller-np59k --tail=5 
84.138.192.89 - - [28/Mar/2020:12:55:34 +0000] "POST /api HTTP/2.0" 200 16 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:74.0) Gecko/20100101 Firefox/74.0" 72 0.834 [default-earth-ratings-80] [] 192.168.99.12:80 16 0.836 200 06f0579ae0775d73858965b45eb32750
2020/03/28 12:56:00 [error] 5548#5548: *119748173 connect() failed (111: Connection refused) while connecting to upstream, client: 91.64.175.187, server: transit-radar.njk.onl, request: "GET / HTTP/2.0", upstream: "http://192.168.99.20:80/", host: "transit-radar.njk.onl"
2020/03/28 12:56:00 [error] 5548#5548: *119748173 connect() failed (111: Connection refused) while connecting to upstream, client: 91.64.175.187, server: transit-radar.njk.onl, request: "GET / HTTP/2.0", upstream: "http://192.168.99.20:80/", host: "transit-radar.njk.onl"
2020/03/28 12:56:00 [error] 5548#5548: *119748173 connect() failed (111: Connection refused) while connecting to upstream, client: 91.64.175.187, server: transit-radar.njk.onl, request: "GET / HTTP/2.0", upstream: "http://192.168.99.20:80/", host: "transit-radar.njk.onl"
91.64.175.187 - - [28/Mar/2020:12:56:00 +0000] "GET / HTTP/2.0" 502 163 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:74.0) Gecko/20100101 Firefox/74.0" 211 0.009 [default-transit-radar-80] [] 192.168.99.20:80, 192.168.99.20:80, 192.168.99.20:80 0, 0, 0 0.000, 0.004, 0.000 502, 502, 502 b851d9a6a1a160542496e8c350e63e6b

# that IP is the IP that the svc picked up
% kubectl describe svc transit-radar
Name:              transit-radar
Namespace:         default
Labels:            app=transit-radar
Annotations:       kubectl.kubernetes.io/last-applied-configuration:
                     {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"app":"transit-radar"},"name":"transit-radar","namespace":"defa...
Selector:          app=transit-radar
Type:              ClusterIP
IP:                10.111.90.182
Port:              <unset>  80/TCP
TargetPort:        80/TCP
Endpoints:         192.168.99.20:80
Session Affinity:  None
Events:            <none>

# and the pod config
% kubectl describe pod -lapp=transit-radar
Name:         transit-radar-7fdc6ccfdd-hk6v5
Namespace:    default
Priority:     0
Node:         k2/10.19.8.53
Start Time:   Fri, 27 Mar 2020 18:56:38 +0100
Labels:       app=transit-radar
              pod-template-hash=7fdc6ccfdd
Annotations:  cni.projectcalico.org/podIP: 192.168.99.20/32
Status:       Running
IP:           192.168.99.20
IPs:
  IP:           192.168.99.20
Controlled By:  ReplicaSet/transit-radar-7fdc6ccfdd
Containers:
  transit-radar:
    Container ID:   docker://211ecc4d1e1675673b570f4bcd46ccd7a88f88a493cc18692328b2c3beb78bd5
    Image:          rg.nl-ams.scw.cloud/njkonl/transit-radar:0.3
    Image ID:       docker-pullable://rg.nl-ams.scw.cloud/njkonl/transit-radar@sha256:94c7fbba4911a8bab9c6132fc3bc11d24921f13ca52331c9c14f250eea6adc33
    Port:           80/TCP
    Host Port:      0/TCP
    State:          Running
      Started:      Fri, 27 Mar 2020 18:56:49 +0100
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     1100m
      memory:  128Mi
    Requests:
      cpu:        10m
      memory:     80Mi
    Environment:  <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from default-token-tz7pr (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  default-token-tz7pr:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  default-token-tz7pr
    Optional:    false
QoS Class:       Burstable
Node-Selectors:  <none>
Tolerations:     node.kubernetes.io/not-ready:NoExecute for 300s
                 node.kubernetes.io/unreachable:NoExecute for 300s
Events:          <none>

deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: transit-radar
  labels:
    app: transit-radar
spec:
  selector:
    matchLabels:
      app: transit-radar
  template:
    metadata:
      labels:
        app: transit-radar
    spec:
      containers:
      - name: transit-radar
        image: rg.nl-ams.scw.cloud/njkonl/transit-radar:0.3
        resources:
          limits:
            memory: "128Mi"
            cpu: "1100m"
          requests:
            memory: "80Mi"
            cpu: "10m"
        ports:
        - containerPort: 80
      imagePullSecrets:
      - name: regcred

---

apiVersion: v1  
kind: Service
metadata:
  name: transit-radar
  labels:
    app: transit-radar
spec:
  selector:
    app: transit-radar
  ports:
  - port: 80
    targetPort: 80

---

apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
  name: transit-radar
  annotations:
    nginx.ingress.kubernetes.io/rewrite-target: /
    kubernetes.io/ingress.class: "nginx"
spec:
  rules:
  - host: transit.njk.onl
    http:
      paths:
      - path: /
        backend:
          serviceName: transit-radar
          servicePort: 80
  tls:
  - hosts:
    - transit.njk.onl
    secretName: transit-radar-cert

---

apiVersion: cert-manager.io/v1alpha2
kind: Certificate
metadata:
  name: transit-radar
spec:
  dnsNames:
  - transit.njk.onl
  issuerRef:
    group: cert-manager.io
    kind: ClusterIssuer
    name: letsencrypt-production
  secretName: transit-radar-cert