istio testing

00-test-istio.yaml

# https://istio.io/latest/docs/reference/config/networking/service-entry/
apiVersion: networking.istio.io/v1beta1
kind: ServiceEntry
metadata:
  name: test-service-entry
  namespace: testing
spec:
  hosts:
  - test.service # for host header matching
  addresses:
  - 192.0.2.0/32 # Needs an IP to use for the listener
  #- 127.0.10.1/32 # Tried this but doesn't seem to work (may not be included in NAT rules)
  ports:
  - number: 7070
    name: mesh-service
    protocol: http
  location: MESH_INTERNAL
---
# https://istio.io/latest/docs/reference/config/networking/virtual-service/
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: test.service
  namespace: testing
spec:
  hosts:
  - test.service
  http:
  # Test mutual tls using: curl -vs -H "Host: test.service" http://192.0.2.0:7070/canary9090
  - match:
    - uri:
        prefix: /canary9090
    route:
    - destination:
        host: test-canary
        port:
          number: 9090
  # Test non-mTLS using: curl -vs -H "Host: test.service" http://192.0.2.0:7070/canary9091
  - match:
    - uri:
        prefix: /canary9091
    rewrite:
      uri: "/headers" # Remove or replace the above prefix before proxying upstream
    route:
    - destination:
        host: test-canary
        port:
          number: 9091
  # Test non-mTLS non-canary using: curl -vs -H "Host: test.service" http://192.0.2.0:7070/app9091
  - match:
    - uri:
        prefix: /app9091
    route:
    - destination:
        host: test
        port:
          number: 9091
  # Fall thru to a weighted test and test-canary (could also use subsets)
  - route:
    - destination:
        host: test-canary #or test-canary.testing.svc.cluster.local
        #subset: v2
        port:
          number: 9090
      weight: 25
    - destination:
        host: test # or test.testing.svc.cluster.local
        #subset: v1
        port:
          number: 9090
      weight: 75
---
# https://istio.io/latest/docs/reference/config/networking/destination-rule/
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: test-canary-dr
  namespace: testing
spec:
# new feature in 1.14 https://istio.io/v1.14/docs/reference/config/networking/destination-rule/
#  workloadSelector: 
#    matchLabels:
#      app: test-canary
  host: test-canary.testing.svc.cluster.local
  trafficPolicy:
    portLevelSettings:
    - port:
        number: 9091
      tls:
        mode: DISABLE
---
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: test-dr
  namespace: testing
spec:
  host: test.testing.svc.cluster.local
  trafficPolicy:
    portLevelSettings:
    - port:
        number: 9091
      tls:
        mode: DISABLE

01-test-app.yaml

# Source https://github.com/GoogleCloudPlatform/kubernetes-engine-samples/tree/main/whereami
apiVersion: apps/v1
kind: Deployment
metadata:
  name: test
  namespace: testing
  labels:
    app: test
spec:
  replicas: 1
  selector:
    matchLabels:
      app: test
  template:
    metadata:
      annotations:
        sidecar.istio.io/inject: "true"
        traffic.sidecar.istio.io/includeInboundPorts: '8080'
        #traffic.sidecar.istio.io/excludeInboundPorts: '1234'
        #traffic.sidecar.istio.io/includeOutboundIPRanges: ""    # Exclude all IPs by default
        #traffic.sidecar.istio.io/includeOutboundPorts: '9090'   # Then always include specific outbound ports to opt-in services to the mesh
        #traffic.sidecar.istio.io/excludeOutboundPorts: '9900,9999'
      labels:
        app: test
    spec:
      containers:
      - name: test-include
        image: us-docker.pkg.dev/google-samples/containers/gke/whereami:v1.2.8
        ports:
        - containerPort: 8080
        env:
          - name: ECHO_HEADERS
            value: "True"
          - name: METADATA
            value: "included inbound port"
      - name: test-exclude
        image: us-docker.pkg.dev/google-samples/containers/gke/whereami:v1.2.8
        ports:
        - containerPort: 8081
        env:
          - name: ECHO_HEADERS
            value: "True"
          - name: PORT
            value: "8081"
          - name: METADATA
            value: "excluded inbound port"
---
apiVersion: "v1"
kind: "Service"
metadata:
  name: "test"
  namespace: testing
spec:
  ports:
  - port: 9090
    targetPort: 8080
    name: http # default http port
  - port: 9091
    targetPort: 8081
    name: http-alternate # protocol-name format
  selector:
    app: "test"
  type: "ClusterIP"
---
# Don't think this works unless you use workload selectors https://istio.io/latest/docs/reference/config/security/peer_authentication/
# This may only be relevant for 1.14 since DestinationRule doesn't have WorloadSelector until then
apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
  name: test
  namespace: testing
spec:
  selector:
    matchLabels:
      app: test
  mtls:
    mode: UNSET
  portLevelMtls:
    # Require mTLS for non-canary. Not sure if it needs svc or target port so will try both
    8080:
      mode: STRICT
    # Disable mTLS on port that isn't using inbound envoy
    8081:
      mode: DISABLE

02-test-canary.yaml

# Pretty much exactly the same as test-app, except PeerAuthentication is permissive
apiVersion: apps/v1
kind: Deployment
metadata:
  name: test-canary
  namespace: testing
  labels:
    app: test-canary
spec:
  replicas: 1
  selector:
    matchLabels:
      app: test-canary
  template:
    metadata:
      annotations:
        sidecar.istio.io/inject: "true"
        traffic.sidecar.istio.io/includeInboundPorts: '8080'
        #traffic.sidecar.istio.io/excludeInboundPorts: '1234'
        #traffic.sidecar.istio.io/includeOutboundIPRanges: ""    # Exclude all IPs by default
        #traffic.sidecar.istio.io/includeOutboundPorts: '9090'   # Then always include specific outbound ports to opt-in services to the mesh
        #traffic.sidecar.istio.io/excludeOutboundPorts: '9900,9999'
      labels:
        app: test-canary
    spec:
      containers:
      - name: test-include
        image: us-docker.pkg.dev/google-samples/containers/gke/whereami:v1.2.8
        ports:
        - containerPort: 8080
        env:
          - name: ECHO_HEADERS
            value: "True"
          - name: METADATA
            value: "canary included inbound port"
      - name: test-exclude
        image: us-docker.pkg.dev/google-samples/containers/gke/whereami:v1.2.8
        ports:
        - containerPort: 8081
        env:
          - name: ECHO_HEADERS
            value: "True"
          - name: PORT
            value: "8081"
          - name: METADATA
            value: "canary excluded inbound port"
---
apiVersion: "v1"
kind: "Service"
metadata:
  name: "test-canary"
  namespace: testing
spec:
  ports:
  - port: 9090
    targetPort: 8080
    name: http # default http port
  - port: 9091
    targetPort: 8081
    name: http-alternate # protocol-name format
  selector:
    app: "test-canary"
  type: "ClusterIP"
---
# This may only be relevant for 1.14 since DestinationRule doesn't have WorloadSelector until then
apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
  name: test-canary
  namespace: testing
spec:
  selector:
    matchLabels:
      app: test-canary
  mtls:
    mode: UNSET
  portLevelMtls:
    # Optional mTLS for canary
    9090:
      mode: PERMISSIVE
    # Disable mTLS on port that isn't using inbound envoy
    9091:
      mode: DISABLE

03-results.sh

# From a container with sidecar using:
traffic.sidecar.istio.io/includeOutboundIPRanges: ""              # Exclude all IPs by default
traffic.sidecar.istio.io/includeOutboundPorts: '9090,9091,7070'   # Then always include specific outbound ports to opt-in services to the mesh

# This uses ServiceEntry and included outbound port to land on sidecar, which then forwards to test-canary port 9090
$ curl -vs -H "Host: test.service" http://192.0.2.0:7070/canary9090
*   Trying 192.0.2.0:7070...
* Connected to 192.0.2.0 (192.0.2.0) port 7070 (#0)
> GET /canary9090 HTTP/1.1
> Host: test.service
> User-Agent: curl/7.81.0
> Accept: */*
> 
* Mark bundle as not supporting multiuse
< HTTP/1.1 200 OK
< server: envoy
< date: Fri, 17 Jun 2022 21:16:09 GMT
< content-type: application/json
< content-length: 1011
< access-control-allow-origin: *
< x-envoy-upstream-service-time: 36
< 
{
  "cluster_name": "gke-oregon", 
  "headers": {
    "Accept": "*/*", 
    "Host": "test.service", 
    "User-Agent": "curl/7.81.0", 
    "X-B3-Parentspanid": "04954a981d1063fa", 
    "X-B3-Sampled": "0", 
    "X-B3-Spanid": "ee26441eec813a7f", 
    "X-B3-Traceid": "caee2f8ecad6399204954a981d1063fa", 
    "X-Envoy-Attempt-Count": "1", 
    "X-Forwarded-Client-Cert": "By=spiffe://gregbray-fleet.svc.id.goog/ns/testing/sa/default;Hash=1a7ae6e1df63d2b946ee51a5465a7fc2f4bf2a1bbea3219e8d67d4e419a6f45c;Subject=\"OU=istio_v1_cloud_workload,O=Google LLC,L=Mountain View,ST=California,C=US\";URI=spiffe://gregbray-fleet.svc.id.goog/ns/testing/sa/default", 
    "X-Forwarded-Proto": "http", 
    "X-Request-Id": "079442ee-9eb0-4324-82e9-400261f851c8"
  }, 
  "host_header": "test.service", 
  "metadata": "canary included inbound port", 
  "pod_name": "test-canary-59ffd599bb-ckk8x", 
  "pod_name_emoji": "🤚🏻", 
  "project_id": "gregbray-vpc", 
  "timestamp": "2022-06-17T21:16:09", 
  "zone": "us-west1-b"
}
* Connection #0 to host 192.0.2.0 left intact

# show just the headers (due to path rewrite) and uses a backend that isn't part of inboundPorts (have to use portLevelSettings to disable mTLS)
# The X-Envoy-Peer headers are what you would see for client using envoy sidecar calling to a server without (they usually are removed by server envoy sidecar)
curl -vs -H "Host: test.service" http://192.0.2.0:7070/canary9091
*   Trying 192.0.2.0:7070...
* Connected to 192.0.2.0 (192.0.2.0) port 7070 (#0)
> GET /canary9091 HTTP/1.1
> Host: test.service
> User-Agent: curl/7.81.0
> Accept: */*
> 
* Mark bundle as not supporting multiuse
< HTTP/1.1 200 OK
< server: envoy
< date: Fri, 17 Jun 2022 21:19:16 GMT
< content-type: application/json
< content-length: 1742
< access-control-allow-origin: *
< x-envoy-upstream-service-time: 32
< 
{
  "Accept": "*/*", 
  "Host": "test.service", 
  "User-Agent": "curl/7.81.0", 
  "X-B3-Sampled": "0", 
  "X-B3-Spanid": "e90cba743f1bed2e", 
  "X-B3-Traceid": "450e73b347866e39e90cba743f1bed2e", 
  "X-Envoy-Attempt-Count": "1", 
  "X-Envoy-Decorator-Operation": "test-canary.testing.svc.cluster.local:9091/canary9091*", 
  "X-Envoy-Original-Path": "/canary9091", 
  "X-Envoy-Peer-Metadata": "ChwKDkFQUF9DT05UQUlORVJTEgoaCGZyb250ZW5kCjMKCkNMVVNURVJfSUQSJRojY24tZ3JlZ2JyYXktdnBjLXVzLXdlc3QxLWdrZS1vcmVnb24KHwoNSVNUSU9fVkVSU0lPThIOGgwxLjExLjgtYXNtLjQKygEKBkxBQkVMUxK/ASq8AQoRCgNhcHASChoId2hlcmVhbWkKIQoRcG9kLXRlbXBsYXRlLWhhc2gSDBoKNTdiNWNjZGRiOAokChlzZWN1cml0eS5pc3Rpby5pby90bHNNb2RlEgcaBWlzdGlvCi0KH3NlcnZpY2UuaXN0aW8uaW8vY2Fub25pY2FsLW5hbWUSChoId2hlcmVhbWkKLwojc2VydmljZS5pc3Rpby5pby9jYW5vbmljYWwtcmV2aXNpb24SCBoGbGF0ZXN0Ch4KB01FU0hfSUQSExoRcHJvai01MDMwNzYyMjcyMzAKIwoETkFNRRIbGhl3aGVyZWFtaS01N2I1Y2NkZGI4LXdwcjZxChYKCU5BTUVTUEFDRRIJGgd0ZXN0aW5nCkwKBU9XTkVSEkMaQWt1YmVybmV0ZXM6Ly9hcGlzL2FwcHMvdjEvbmFtZXNwYWNlcy90ZXN0aW5nL2RlcGxveW1lbnRzL3doZXJlYW1pCskCChFQTEFURk9STV9NRVRBREFUQRKzAiqwAgosChNnY3BfZ2NlX2luc3RhbmNlX2lkEhUaEzU5ODQ5MzA0NTc4OTA5NDU4MjAKJAoUZ2NwX2drZV9jbHVzdGVyX25hbWUSDBoKZ2tlLW9yZWdvbgp5ChNnY3BfZ2tlX2NsdXN0ZXJfdXJsEmIaYGh0dHBzOi8vY29udGFpbmVyLmdvb2dsZWFwaXMuY29tL3YxL3Byb2plY3RzL2dyZWdicmF5LXZwYy9sb2NhdGlvbnMvdXMtd2VzdDEvY2x1c3RlcnMvZ2tlLW9yZWdvbgoaCgxnY3BfbG9jYXRpb24SChoIdXMtd2VzdDEKHQoLZ2NwX3Byb2plY3QSDhoMZ3JlZ2JyYXktdnBjCiQKEmdjcF9wcm9qZWN0X251bWJlchIOGgw1MDMwNzYyMjcyMzAKGwoNV09SS0xPQURfTkFNRRIKGgh3aGVyZWFtaQ==", 
  "X-Envoy-Peer-Metadata-Id": "sidecar~10.96.4.19~whereami-57b5ccddb8-wpr6q.testing~testing.svc.cluster.local", 
  "X-Forwarded-Proto": "http", 
  "X-Request-Id": "bcb658d8-cae6-4111-8cb2-7489002cd92d"
}

# Skipping output of this, as its essentially the same as /canary9091 but doensn't rewrite the url when sending to the backend
curl -vs -H "Host: test.service" http://192.0.2.0:7070/app9091

# And actually since we aren't using the host header, looks like that is optional. But probably best to include it
# Here is the /metadata endpoint in whereami but using the weighted loadbalancing
$ for i in $(seq 10); do curl -s http://192.0.2.0:7070/metadata;echo ''; done;
included inbound port
canary included inbound port
included inbound port
included inbound port
included inbound port
included inbound port
canary included inbound port
included inbound port
canary included inbound port
included inbound port

Other things you can test:

# using existing k8s service name:port should work (assuming they are exported to your namespace/app and allowed by network policies)
curl -vs http://test.testing.svc.cluster.local:9090/
# Or use port 9091 to omit server side envoy (would need to make sure and disable mTLS from client sidecar though)

# Failure trying to talk to test-app pod directly since it requires mTLS.
curl -vs http://10.96.3.15:8080/
*   Trying 10.96.3.15:8080...
* Connected to 10.96.3.15 (10.96.3.15) port 8080 (#0)
> GET / HTTP/1.1
> Host: 10.96.3.15:8080
> User-Agent: curl/7.81.0
> Accept: */*
> 
* Recv failure: Connection reset by peer
* Closing connection 0

# Switching to 8081 would work since that bypasses server side envoy.
# Or using an IP:8080 for test-canary pod would work since it is in PERMISSIVE mode