Record metrics for short running pods in Rancher Prometheus

metrics-short-running-pods.md

Instructions

First, create a custom ServiceMonitor resource in the cattle-prometheus namespace of the corresponding cluster. This ServiceMonitor will make Prometheus scrape the resource metrics for all Pods that have a matching annotation of prom_scrape_every_5s: "true" every 5 seconds.

$ kubectl create -n cattle-prometheus -f sm.yaml

The contents of the sm.yaml file:

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: exporter-kubelets-cluster-monitoring-custom
  namespace: cattle-prometheus
spec:
  endpoints:
  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    honorLabels: true
    interval: 5s
    metricRelabelings:
    # scrape only pods that have prom_scrape_every_5s: "true"' annotation
    - action: keep
      regex: "true"
      sourceLabels:
      - __meta_kubernetes_pod_annotation_prom_scrape_every_5s
    - action: replace
      regex: (.+)
      replacement: $1
      sourceLabels:
      - container
      targetLabel: container_name
    - action: replace
      regex: (.+)
      replacement: $1
      sourceLabels:
      - pod
      targetLabel: pod_name
    path: /metrics/resource/v1alpha1
    port: https-metrics
    scheme: https
    tlsConfig:
      caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      insecureSkipVerify: true
  namespaceSelector:
    matchNames:
    - cattle-prometheus
  selector:
    matchLabels:
      k8s-app: kubelet

To test, deploy a Pod resource with matching annotation:

apiVersion: v1
kind: Pod
metadata:
  name: memory-consumer
  annotations:
    prom_scrape_every_5s: "true"
spec:
  restartPolicy: Never
  containers:
  - image: polinux/stress
    name: memory-consumer
    args:
    - --vm
    - "1"
    - --vm-bytes
    - 128M
    - --timeout
    - 10s
    command:
    - stress