Istio OTEL test

agent.yaml

traces:
    configs:
      - automatic_logging:
          backend: stdout
          loki_name: default
          spans: true
        batch:
            send_batch_size: 1000
            timeout: 5s
        name: default
        receivers:
            otlp:
                protocols:
                    grpc: null
        remote_write:
          - basic_auth:
                password: ${TOKEN}
                username: ${USER}
            endpoint: tempo-eu-west-0.grafana.net:443
            retry_on_failure:
                enabled: false
        scrape_configs:
          - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
            job_name: kubernetes-pods
            kubernetes_sd_configs:
              - role: pod
            relabel_configs:
              - action: replace
                source_labels:
                  - __meta_kubernetes_namespace
                target_label: namespace
              - action: replace
                source_labels:
                  - __meta_kubernetes_pod_name
                target_label: pod
              - action: replace
                source_labels:
                  - __meta_kubernetes_pod_container_name
                target_label: container
            tls_config:
                ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
                insecure_skip_verify: false

isotope.yaml

apiVersion: v1
kind: Namespace
metadata:
  name: scenario
  labels:
    istio-injection: enabled
---
apiVersion: v1
data:
  service-graph.yaml: |
    apiVersion: v1alpha1
    kind: MockServiceGraph
    defaults:
      type: http
      requestSize: 16 KB
      isEntrypoint: false
      responseSize: 16 KB

    services:
    - name: gateway
      errorRate: 0%
      isEntrypoint: true
      script:
      - - call: svc1
        - call: svc2
    # 1st subtree
    - name: svc1
      errorRate: 2%
      script:
      - call: svc3
    - name: svc3
      errorRate: 5%
    # 2nd subtree
    - name: svc2
      errorRate: 5%
      script:
      - - call: svc4
        - call: svc5
    - name: svc4
      errorRate: 0%
    - name: svc5
      errorRate: 10%
      script:
      - call: svc6
    - name: svc6
      errorRate: 10%
kind: ConfigMap
metadata:
  name: config
  namespace: scenario
---
apiVersion: v1
kind: Service
metadata:
  labels:
    lightpath.cloud/proxy: disabled
    name: gateway
  name: gateway
  namespace: scenario
spec:
  ports:
  - name: http
    port: 8080
    targetPort: 8080
  selector:
    name: gateway
  type: NodePort
---
apiVersion: v1
kind: Service
metadata:
  labels:
    lightpath.cloud/proxy: disabled
    name: svc1
  name: svc1
  namespace: scenario
spec:
  ports:
  - name: http
    port: 8080
    targetPort: 8080
  selector:
    name: svc1
---
apiVersion: v1
kind: Service
metadata:
  labels:
    lightpath.cloud/proxy: disabled
    name: svc2
  name: svc2
  namespace: scenario
spec:
  ports:
  - name: http
    port: 8080
    targetPort: 8080
  selector:
    name: svc2
---
apiVersion: v1
kind: Service
metadata:
  labels:
    lightpath.cloud/proxy: disabled
    name: svc3
  name: svc3
  namespace: scenario
spec:
  ports:
  - name: http
    port: 8080
    targetPort: 8080
  selector:
    name: svc3
---
apiVersion: v1
kind: Service
metadata:
  labels:
    lightpath.cloud/proxy: disabled
    name: svc4
  name: svc4
  namespace: scenario
spec:
  ports:
  - name: http
    port: 8080
    targetPort: 8080
  selector:
    name: svc4
---
apiVersion: v1
kind: Service
metadata:
  labels:
    lightpath.cloud/proxy: disabled
    name: svc5
  name: svc5
  namespace: scenario
spec:
  ports:
  - name: http
    port: 8080
    targetPort: 8080
  selector:
    name: svc5
---
apiVersion: v1
kind: Service
metadata:
  labels:
    lightpath.cloud/proxy: disabled
    name: svc6
  name: svc6
  namespace: scenario
spec:
  ports:
  - name: http
    port: 8080
    targetPort: 8080
  selector:
    name: svc6
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: gateway
  namespace: scenario
spec:
  minReadySeconds: 10
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      name: gateway
  template:
    metadata:
      annotations:
        config-hash: 36dbacac8ece29e57e7573800a6105eb
        prometheus.io/path: /metrics
        prometheus.io/port: "8080"
        prometheus.io/scrape: "true"
      labels:
        name: gateway
    spec:
      containers:
      - env:
        - name: SERVICE_NAME
          value: gateway
        image: ghcr.io/xvzf/isotope:otel-header-propagation-2
        imagePullPolicy: Always
        name: performance-test
        ports:
        - containerPort: 8080
          name: http
        volumeMounts:
        - mountPath: /etc/config/
          name: config
      volumes:
      - configMap:
          name: config
        name: config
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: svc1
  namespace: scenario
spec:
  minReadySeconds: 10
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      name: svc1
  template:
    metadata:
      annotations:
        config-hash: 36dbacac8ece29e57e7573800a6105eb
        prometheus.io/path: /metrics
        prometheus.io/port: "8080"
        prometheus.io/scrape: "true"
      labels:
        name: svc1
    spec:
      containers:
      - env:
        - name: SERVICE_NAME
          value: svc1
        image: ghcr.io/xvzf/isotope:otel-header-propagation-2
        imagePullPolicy: Always
        name: performance-test
        ports:
        - containerPort: 8080
          name: http
        volumeMounts:
        - mountPath: /etc/config/
          name: config
      volumes:
      - configMap:
          name: config
        name: config
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: svc2
  namespace: scenario
spec:
  minReadySeconds: 10
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      name: svc2
  template:
    metadata:
      annotations:
        config-hash: 36dbacac8ece29e57e7573800a6105eb
        prometheus.io/path: /metrics
        prometheus.io/port: "8080"
        prometheus.io/scrape: "true"
      labels:
        name: svc2
    spec:
      containers:
      - env:
        - name: SERVICE_NAME
          value: svc2
        image: ghcr.io/xvzf/isotope:otel-header-propagation-2
        imagePullPolicy: Always
        name: performance-test
        ports:
        - containerPort: 8080
          name: http
        volumeMounts:
        - mountPath: /etc/config/
          name: config
      volumes:
      - configMap:
          name: config
        name: config
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: svc3
  namespace: scenario
spec:
  minReadySeconds: 10
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      name: svc3
  template:
    metadata:
      annotations:
        config-hash: 36dbacac8ece29e57e7573800a6105eb
        prometheus.io/path: /metrics
        prometheus.io/port: "8080"
        prometheus.io/scrape: "true"
      labels:
        name: svc3
    spec:
      containers:
      - env:
        - name: SERVICE_NAME
          value: svc3
        image: ghcr.io/xvzf/isotope:otel-header-propagation-2
        imagePullPolicy: Always
        name: performance-test
        ports:
        - containerPort: 8080
          name: http
        volumeMounts:
        - mountPath: /etc/config/
          name: config
      volumes:
      - configMap:
          name: config
        name: config
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: svc4
  namespace: scenario
spec:
  minReadySeconds: 10
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      name: svc4
  template:
    metadata:
      annotations:
        config-hash: 36dbacac8ece29e57e7573800a6105eb
        prometheus.io/path: /metrics
        prometheus.io/port: "8080"
        prometheus.io/scrape: "true"
      labels:
        name: svc4
    spec:
      containers:
      - env:
        - name: SERVICE_NAME
          value: svc4
        image: ghcr.io/xvzf/isotope:otel-header-propagation-2
        imagePullPolicy: Always
        name: performance-test
        ports:
        - containerPort: 8080
          name: http
        volumeMounts:
        - mountPath: /etc/config/
          name: config
      volumes:
      - configMap:
          name: config
        name: config
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: svc5
  namespace: scenario
spec:
  minReadySeconds: 10
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      name: svc5
  template:
    metadata:
      annotations:
        config-hash: 36dbacac8ece29e57e7573800a6105eb
        prometheus.io/path: /metrics
        prometheus.io/port: "8080"
        prometheus.io/scrape: "true"
      labels:
        name: svc5
    spec:
      containers:
      - env:
        - name: SERVICE_NAME
          value: svc5
        image: ghcr.io/xvzf/isotope:otel-header-propagation-2
        imagePullPolicy: Always
        name: performance-test
        ports:
        - containerPort: 8080
          name: http
        volumeMounts:
        - mountPath: /etc/config/
          name: config
      volumes:
      - configMap:
          name: config
        name: config
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: svc6
  namespace: scenario
spec:
  minReadySeconds: 10
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      name: svc6
  template:
    metadata:
      annotations:
        config-hash: 36dbacac8ece29e57e7573800a6105eb
        prometheus.io/path: /metrics
        prometheus.io/port: "8080"
        prometheus.io/scrape: "true"
      labels:
        name: svc6
    spec:
      containers:
      - env:
        - name: SERVICE_NAME
          value: svc6
        image: ghcr.io/xvzf/isotope:otel-header-propagation-2
        imagePullPolicy: Always
        name: performance-test
        ports:
        - containerPort: 8080
          name: http
        volumeMounts:
        - mountPath: /etc/config/
          name: config
      volumes:
      - configMap:
          name: config
        name: config
---
apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
  name: default
  namespace: scenario
spec:
  selector:
    istio: ingressgateway # use istio default controller
  servers:
  - port:
      number: 80
      name: http
      protocol: HTTP
    hosts:
    - "*"
---
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: gateway
  namespace: scenario
spec:
  gateways:
  - default
  hosts:
  - '*'
  http:
  - name: "default"
    route:
    - destination:
        host: gateway.scenario.svc.cluster.local
        port:
          number: 8080

istio-operator.yaml

# vim: set ft=yaml:
# This is a reduced version of https://github.com/ankorstore/infrastructure/blob/main/terraform/modules/core/templates/istio-operator.yaml.tpl
# aligned to be as close to production as possible!
---
apiVersion: install.istio.io/v1alpha1
kind: IstioOperator
spec:
  components:
    ingressGateways:
      - enabled: true
        name: istio-ingressgateway
    pilot:  # This is actually istiod
      enabled: true
  meshConfig:
    extensionProviders:
      - name: otel
        opentelemetry:
          service: grafana-agent-traces.grafana-agent.svc.cluster.local
          port: 4317
    defaultConfig:
      proxyMetadata:
        # Intercept DNS traffic for direct returns avoiding round-trips + resolving internal, non DNS exposed domains
        ISTIO_META_DNS_CAPTURE: "true"
        ISTIO_META_DNS_AUTO_ALLOCATE: "true"
      gatewayTopology:
        numTrustedProxies: 2  # Two google IP redirects
    accessLogEncoding: JSON  # Easy to parse with e.g. Loki
    accessLogFile: /dev/stdout
    accessLogFormat: |
      {
        "authority": "%REQ(:AUTHORITY)%",
        "bytes_received": "%BYTES_RECEIVED%",
        "bytes_sent": "%BYTES_SENT%",
        "client_ip": "%REQ(True-Client-Ip)%",
        "method": "%REQ(:METHOD)%",
        "path": "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%",
        "protocol": "%PROTOCOL%",
        "request_id": "%REQ(X-REQUEST-ID)%",
        "response_code": "%RESPONSE_CODE%",
        "response_code_details": "%RESPONSE_CODE_DETAILS%",
        "traceparent": "%REQ(traceparent)%",
        "upstream_cluster": "%UPSTREAM_CLUSTER%",
        "upstream_host": "%UPSTREAM_HOST%",
        "upstream_local_address": "%UPSTREAM_LOCAL_ADDRESS%"
      }
    # While not being optimal for now, chaning this results in too much overhead we cannot
    # afford right now.
    #
    # Can be optimised in the future by exporing to proxies running in the same namespace only
    # and only export public-facing services to the `istio-system` namespace where the IGW requires endpoint information
    defaultDestinationRuleExportTo:
      - '*'
    defaultServiceExportTo:
      - '*'
    defaultVirtualServiceExportTo:
      - '*'
    enablePrometheusMerge: true  # Only one metric endpoint to scrape, pulls metrics from the app and merges it with envoy metrics
  values:
    gateways:
      # Network config magic begins here!
      istio-ingressgateway:
        type: NodePort # Expose istio endpoints as nodeport, check here for the configuration: https://istio.io/v1.5/docs/reference/config/installation-options/#gateways-options
        ports:
        - name: status-port
          port: 15021
          protocol: TCP
          targetPort: 15021
        - name: http2
          port: 80
          protocol: TCP
          targetPort: 8080
          nodePort: 30080
        - name: https
          port: 443
          protocol: TCP
          targetPort: 8443
          nodePort: 30443
    global:
      logAsJson: true
      logging:
        level: default:debug  # Helpful information when sth goes wrong, not very noisy
      proxy:
        autoInject: enabled  # Enable by default, disable by pod annotation
        logLevel: info  # We want logs about xDS API updates when tracking down issueslogs about xDS API updates when tracking down issues

telemetry.yaml

apiVersion: telemetry.istio.io/v1alpha1
kind: Telemetry
metadata:
  name: otel-tracing
  namespace: istio-system
spec:
  tracing:
  - providers:
    - name: otel
    randomSamplingPercentage: 100.0