OTEL collector deployment discovering prometheus endpoints à-la nri-prometheus

gateway.yml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: gateway-conf
  namespace: otel
data:
  otel-gateway-config.yaml: |
    receivers:
      otlp:
        protocols:
          grpc:
          http:
    processors:
      batch:
    exporters:
      otlp:
        endpoint: staging-otlp.nr-data.net:4317
        headers:
          api-key: ${NR_API_KEY}
      prometheus:
        endpoint: ":8081" # Expose labeled prometheus metrics in this port
        resource_to_telemetry_conversion:
          enabled: true
    extensions:
      health_check:
    service:
      extensions: [ health_check ]
      pipelines:
        metrics:
          receivers: [ otlp ]
          processors: [ batch ]
          #exporters: [ otlp, prometheus ]
          exporters: [ prometheus ]
---
apiVersion: v1
kind: Service
metadata:
  name: gateway
  namespace: otel
spec:
  ports:
    - name: otlp # Default endpoint for OpenTelemetry receiver.
      port: 4317
      protocol: TCP
      targetPort: 4317
    - name: prometheus-labeled
      port: 8081
      protocol: TCP
      targetPort: 8081
  selector:
    component: otel-gateway
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: gateway
  namespace: otel
  labels:
    app: opentelemetry
    component: otel-gateway
spec:
  selector:
    matchLabels:
      app: opentelemetry
      component: otel-gateway
  minReadySeconds: 5
  progressDeadlineSeconds: 120
  replicas: 1
  template:
    metadata:
      labels:
        app: opentelemetry
        component: otel-gateway
    spec:
      containers:
        - command:
            - "/otelcontribcol"
            - "--config=/conf/otel-gateway-config.yaml"
          image: otel/opentelemetry-collector-contrib
          name: otel-gateway
          env:
            - name: NR_API_KEY
              valueFrom:
                secretKeyRef:
                  key: NR_API_KEY
                  name: nr-secret
          resources:
            limits:
              cpu: 200m
              memory: 500M
            requests:
              cpu: 10m
              memory: 100Mi
          ports:
            - containerPort: 4317 # Default endpoint for OpenTelemetry receiver.
            - containerPort: 8081 # Prometheus exporter
          volumeMounts:
            - name: otel-gateway-config-vol
              mountPath: /conf
          livenessProbe:
            httpGet:
              path: /
              port: 13133 # Health Check extension default port.
          readinessProbe:
            httpGet:
              path: /
              port: 13133 # Health Check extension default port.
      volumes:
        - configMap:
            name: gateway-conf
            items:
              - key: otel-gateway-config.yaml
                path: otel-gateway-config.yaml
          name: otel-gateway-config-vol

promths.yml

apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-conf
  namespace: otel
data:
  otel-prometheus-config.yaml: |
    receivers:
      prometheus:
        config:
          scrape_configs:
            - job_name: k8s_newrelic_scrape
              scrape_interval: 45s
              kubernetes_sd_configs:
                # Attempt to scrape endpoints (pods behind annotated services)
                - role: endpoints
                # And also pods that are annotated directly and are not behind a service
                - role: pod
              relabel_configs:
                # Multiple instances of `relabel_configs` are applied sequentially. If with action:keep does not match
                # a target, it will be dropped immediately and subsequent configs will be noop.
                - source_labels:
                  # Multiple source_labels are concatenated together with `;` before checking if regex matches
                  # By specifying a permissive regex we achieve a shitty OR, matching e.g `;;true;`
                  - __meta_kubernetes_pod_label_newrelic_com_scrape
                  - __meta_kubernetes_pod_annotation_newrelic_com_scrape
                  - __meta_kubernetes_service_label_newrelic_com_scrape
                  - __meta_kubernetes_service_annotation_newrelic_com_scrape
                  # We need to add .* in both ends to get the shitty ORing because prometheus wraps this regex
                  # with ^ $ automatically, resulting in ^.*true.*$
                  regex: ".*true.*"
                  action: keep
            # Extra job for headless services (with no endpoints behind)
            - job_name: k8s_newrelic_scrape_headless
              scrape_interval: 45s
              kubernetes_sd_configs:
                - role: service
              relabel_configs:
                - source_labels:
                  - __meta_kubernetes_service_label_newrelic_com_scrape
                  - __meta_kubernetes_service_annotation_newrelic_com_scrape
                  regex: ".*true.*"
                  action: keep
                # Drop services that have an empty external_name
                - source_labels:
                  - __meta_kubernetes_service_external_name
                  regex: ".+"
                  action: keep
    processors:
      batch: {}
    exporters:
      otlp:
        endpoint: gateway.otel.svc:4317
        insecure: true
      logging:
        loglevel: debug
      prometheus:
        endpoint: ":8081"
        resource_to_telemetry_conversion:
          enabled: true
    extensions:
      health_check:
    service:
      extensions: [ health_check ]
      pipelines:
        metrics:
          receivers: [ prometheus ]
          processors: [ batch ]
          exporters: [ otlp, prometheus ]
---
apiVersion: v1
kind: Service
metadata:
  name: prometheus
  namespace: otel
spec:
  ports:
    - name: prometheus-labeled
      port: 8081
      protocol: TCP
      targetPort: 8081
  selector:
    component: otel-prometheus
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus
  namespace: otel
  labels:
    app: opentelemetry
    component: otel-prometheus
spec:
  selector:
    matchLabels:
      app: opentelemetry
      component: otel-prometheus
  minReadySeconds: 5
  progressDeadlineSeconds: 120
  replicas: 1
  template:
    metadata:
      labels:
        app: opentelemetry
        component: otel-prometheus
    spec:
      serviceAccountName: otel-prometheus-sa
      containers:
        - command:
            - "/otelcontribcol"
            - "--config=/conf/otel-prometheus-config.yaml"
          image: otel/opentelemetry-collector-contrib
          name: otel-prometheus
          resources:
            limits:
              cpu: 200m
              memory: 500M
            requests:
              cpu: 10m
              memory: 100Mi
          volumeMounts:
            - name: otel-prometheus-config-vol
              mountPath: /conf
          livenessProbe:
            httpGet:
              path: /
              port: 13133 # Health Check extension default port.
          readinessProbe:
            httpGet:
              path: /
              port: 13133 # Health Check extension default port.
      volumes:
        - configMap:
            name: prometheus-conf
            items:
              - key: otel-prometheus-config.yaml
                path: otel-prometheus-config.yaml
          name: otel-prometheus-config-vol
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: otel-prometheus-sa
  namespace: otel
  labels:
    app: otel-prometheus
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: otel-prometheus-role
  namespace: otel
rules:
  - apiGroups:
      - ""
    resources:
      - nodes
      - pods
      - services
      - endpoints
    verbs:
      - get
      - list
      - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: read-otel-prometheus-role
  namespace: otel
subjects:
  - kind: ServiceAccount
    name: otel-prometheus-sa
    namespace: otel
roleRef:
  kind: ClusterRole
  name: otel-prometheus-role
  apiGroup: rbac.authorization.k8s.io