Linkerd 503 Service Unavailable repro

ingress-nginx.yaml

apiVersion: v1
kind: Namespace
metadata:
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
  name: ingress-nginx
---

kind: ConfigMap
apiVersion: v1
metadata:
  name: nginx-configuration
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
kind: ConfigMap
apiVersion: v1
metadata:
  name: tcp-services
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
data:
  5044: "default/logstash:5044"
  8080: "emojivoto/web-svc:http"
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: udp-services
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: nginx-ingress-serviceaccount
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: nginx-ingress-clusterrole
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - endpoints
      - nodes
      - pods
      - secrets
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - services
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - events
    verbs:
      - create
      - patch
  - apiGroups:
      - "extensions"
      - "networking.k8s.io"
    resources:
      - ingresses
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - "extensions"
      - "networking.k8s.io"
    resources:
      - ingresses/status
    verbs:
      - update

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
  name: nginx-ingress-role
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - pods
      - secrets
      - namespaces
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - configmaps
    resourceNames:
      # Defaults to "<election-id>-<ingress-class>"
      # Here: "<ingress-controller-leader>-<nginx>"
      # This has to be adapted if you change either parameter
      # when launching the nginx-ingress-controller.
      - "ingress-controller-leader-nginx"
    verbs:
      - get
      - update
  - apiGroups:
      - ""
    resources:
      - configmaps
    verbs:
      - create
  - apiGroups:
      - ""
    resources:
      - endpoints
    verbs:
      - get

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: nginx-ingress-role-nisa-binding
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: nginx-ingress-role
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: nginx-ingress-clusterrole-nisa-binding
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: nginx-ingress-clusterrole
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx

---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
  name: nginx-ingress-controller
  namespace: ingress-nginx
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  template:
    metadata:
      annotations:
        prometheus.io/port: "10254"
        prometheus.io/scrape: "true"
      labels:
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
    spec:
      containers:
      - args:
        - /nginx-ingress-controller
        - --configmap=$(POD_NAMESPACE)/nginx-configuration
        - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
        - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
        - --publish-service=$(POD_NAMESPACE)/ingress-nginx
        - --annotations-prefix=nginx.ingress.kubernetes.io
        - --v=2
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.31.1
        lifecycle:
          preStop:
            exec:
              command:
              - /wait-shutdown
        livenessProbe:
          failureThreshold: 3
          httpGet:
            path: /healthz
            port: 10254
            scheme: HTTP
          initialDelaySeconds: 10
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 10
        name: nginx-ingress-controller
        ports:
        - containerPort: 80
          hostPort: 80
          name: http
          protocol: TCP
        - containerPort: 443
          hostPort: 443
          name: https
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /healthz
            port: 10254
            scheme: HTTP
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 10
        securityContext:
          allowPrivilegeEscalation: true
          capabilities:
            add:
            - NET_BIND_SERVICE
            drop:
            - ALL
          runAsUser: 101
      nodeSelector:
        ingress-ready: "true"
        kubernetes.io/os: linux
      tolerations:
      - key: node-role.kubernetes.io/master
        operator: Equal
        effect: NoSchedule
      serviceAccountName: nginx-ingress-serviceaccount
      terminationGracePeriodSeconds: 3
---
apiVersion: v1
kind: Service
metadata:
  name: ingress-nginx
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
spec:
  type: NodePort
  ports:
    - name: http
      port: 80
      targetPort: 80
      protocol: TCP
    - name: https
      port: 443
      targetPort: 443
      protocol: TCP
  selector:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

kind-cluster-ingress.yaml

kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
  kubeadmConfigPatches:
  - |
    kind: InitConfiguration
    nodeRegistration:
      kubeletExtraArgs:
        node-labels: "ingress-ready=true"
        authorization-mode: "AlwaysAllow"
  extraPortMappings:
  - containerPort: 80
    hostPort: 80
    protocol: TCP
  - containerPort: 443
    hostPort: 443
    protocol: TCP
- role: worker

kuard.yaml

apiVersion: v1
kind: Service
metadata:
  name: kuard
  namespace: default
spec:
  ports:
  - name: http
    port: 8080
    targetPort: 8080
  selector:
    app: kuard
  type: ClusterIP
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app.kubernetes.io/name: kuard
  name: kuard
  namespace: default
spec:
  replicas: 2
  selector:
    matchLabels:
      app: kuard
  template:
    metadata:
      labels:
        app: kuard
    spec:
      containers:
      - name: kuard
        image: gcr.io/kuar-demo/kuard-amd64:blue
        ports:
        - name: http
          containerPort: 8080
        resources:
          requests:
            cpu: 100m
---
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
  name: kuard
  namespace: default
  annotations:
    nginx.ingress.kubernetes.io/service-upstream: "true"
    nginx.ingress.kubernetes.io/configuration-snippet: |
      proxy_set_header l5d-dst-override $service_name.$namespace.svc.cluster.local:$service_port;
      grpc_set_header  l5d-dst-override $service_name.$namespace.svc.cluster.local:$service_port;
spec:
  rules:
  - host: localhost
    http:
      paths:
      - backend:
          serviceName: kuard
          servicePort: 8080
        path: /

linkerd-proxy-trace.log

linkerd-proxy in nginx-ingress-controller pod log excerpt (trace instead of debug logging):
1820416:[    71.898804100s] TRACE outbound:accept{peer.addr=10.244.0.5:43650}:source{target.addr=10.104.211.127:8080}: linkerd2_router: Making
1820417:[    71.898815111s] TRACE outbound:accept{peer.addr=10.244.0.5:43650}:source{target.addr=10.104.211.127:8080}:logical{addr=kuard.default.svc.cluster.local:8080}: linkerd2_stack_tracing: making
1820418:[    71.898821663s] TRACE outbound:accept{peer.addr=10.244.0.5:43650}:source{target.addr=10.104.211.127:8080}:logical{addr=kuard.default.svc.cluster.local:8080}: linkerd2_router: Responding
1820419:[    71.898828335s] TRACE outbound:accept{peer.addr=10.244.0.5:43650}:source{target.addr=10.104.211.127:8080}:logical{addr=kuard.default.svc.cluster.local:8080}:profile: linkerd2_stack_tracing: poll ready
1820420:[    71.898840628s] TRACE outbound:accept{peer.addr=10.244.0.5:43650}:source{target.addr=10.104.211.127:8080}:logical{addr=kuard.default.svc.cluster.local:8080}:profile: linkerd2_stack_tracing: ready=false
1820421:[    71.898847802s] TRACE outbound:accept{peer.addr=10.244.0.5:43650}:source{target.addr=10.104.211.127:8080}:logical{addr=kuard.default.svc.cluster.local:8080}: linkerd2_stack_tracing: ready=false
1820422:[    71.898854033s]  WARN outbound:accept{peer.addr=10.244.0.5:43650}:source{target.addr=10.104.211.127:8080}: linkerd2_app_core::errors: Failed to proxy request: request timed out
1820423:[    71.898858952s] DEBUG outbound:accept{peer.addr=10.244.0.5:43650}:source{target.addr=10.104.211.127:8080}: linkerd2_app_core::errors: Handling error with HTTP response status=503 Service Unavailable version=HTTP/1.1
1820424:[    71.898866446s] TRACE outbound:accept{peer.addr=10.244.0.5:43650}: linkerd2_concurrency_limit: releasing permit available=9153
1820425:[    71.898880412s] TRACE outbound:accept{peer.addr=10.244.0.5:43650}: hyper::proto::h1::role: Server::encode status=503, body=None, req_method=Some(GET)
1820426:[    71.898901422s] DEBUG outbound:accept{peer.addr=10.244.0.5:43650}: hyper::proto::h1::io: flushed 92 bytes
1820427:[    71.898909447s] TRACE outbound:accept{peer.addr=10.244.0.5:43650}: hyper::proto::h1::conn: maybe_notify; read_from_io blocked
1820428:[    71.898913865s] TRACE outbound:accept{peer.addr=10.244.0.5:43650}: hyper::proto::h1::conn: flushed({role=server}): State { reading: Init, writing: Init, keep_alive: Idle }

# 5000 requests, 1000 workers, 1 rps per worker - disaster
> hey -n 5000 -c 1000 -q 1 'https://localhost/'

Summary:
  Total:	17.7769 secs
  Slowest:	4.8363 secs
  Fastest:	0.1053 secs
  Average:	3.1311 secs
  Requests/sec:	281.2636


Response time histogram:
  0.105 [1]	|
  0.578 [48]	|■
  1.051 [20]	|
  1.525 [20]	|
  1.998 [27]	|
  2.471 [37]	|
  2.944 [251]	|■■■
  3.417 [3763]	|■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■
  3.890 [687]	|■■■■■■■
  4.363 [7]	|
  4.836 [139]	|■


Latency distribution:
  10% in 3.0282 secs
  25% in 3.0663 secs
  50% in 3.0945 secs
  75% in 3.1735 secs
  90% in 3.5255 secs
  95% in 3.6083 secs
  99% in 4.5587 secs

Details (average, fastest, slowest):
  DNS+dialup:	0.0455 secs, 0.1053 secs, 4.8363 secs
  DNS-lookup:	0.0111 secs, 0.0000 secs, 0.1757 secs
  req write:	0.0005 secs, 0.0000 secs, 0.0992 secs
  resp wait:	3.0689 secs, 0.0408 secs, 4.5357 secs
  resp read:	0.0003 secs, 0.0000 secs, 0.1017 secs

Status code distribution:
  [200]	808 responses
  [503]	4192 responses
  
# 5000 requests, 100 workers, 10 rps per worker - all fine
> hey -n 5000 -c 100 -q 10 'https://localhost/'

Summary:
  Total:	18.4520 secs
  Slowest:	0.5218 secs
  Fastest:	0.0435 secs
  Average:	0.3650 secs
  Requests/sec:	270.9740


Response time histogram:
  0.043 [1]	|
  0.091 [4]	|
  0.139 [13]	|
  0.187 [17]	|
  0.235 [23]	|
  0.283 [33]	|
  0.331 [108]	|■
  0.378 [3453]	|■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■
  0.426 [1275]	|■■■■■■■■■■■■■■■
  0.474 [64]	|■
  0.522 [9]	|


Latency distribution:
  10% in 0.3413 secs
  25% in 0.3534 secs
  50% in 0.3662 secs
  75% in 0.3798 secs
  90% in 0.3931 secs
  95% in 0.4048 secs
  99% in 0.4306 secs

Details (average, fastest, slowest):
  DNS+dialup:	0.0010 secs, 0.0435 secs, 0.5218 secs
  DNS-lookup:	0.0001 secs, 0.0000 secs, 0.0541 secs
  req write:	0.0000 secs, 0.0000 secs, 0.0202 secs
  resp wait:	0.3636 secs, 0.0268 secs, 0.4840 secs
  resp read:	0.0002 secs, 0.0001 secs, 0.0052 secs

Status code distribution:
  [200]	5000 responses

run.sh

#!/bin/bash
set -euxo pipefail
kind create cluster --config kind-cluster-ingress.yaml --image=kindest/node:v1.18.2
kubectl wait --for=condition=Ready --timeout=120s node kind-worker
linkerd install | kubectl apply -f -
linkerd check

# ingress-nginx - gets deployed to control plane node
linkerd inject --skip-inbound-ports 80,443 --proxy-log-level debug ingress-nginx.yaml | kubectl apply -f -

# kuard - gets deployed to worker node
linkerd inject kuard.yaml | kubectl apply -f -
kubectl wait --for=condition=Available --timeout=120s deploy kuard

linkerd dashboard & # will show 100% success rate

# go get github.com/rakyll/hey
hey -n 5000 -c 1000 -q 1 'https://localhost/' # will show large amount of 503 responses