Skip to content

Instantly share code, notes, and snippets.

@XciD
Last active March 6, 2019 10:29
Show Gist options
  • Save XciD/341577e907d7a972a5239251bf3f62a7 to your computer and use it in GitHub Desktop.
Save XciD/341577e907d7a972a5239251bf3f62a7 to your computer and use it in GitHub Desktop.
Prometheus Config
# Based on https://github.com/prometheus/prometheus/blob/release-2.2/documentation/examples/prometheus-kubernetes.yml
serverFiles:
prometheus.yml:
remote_write:
- url: "https://prometheus.gra1-ovh.metrics.ovh.net/remote_write"
remote_timeout: 120s
bearer_token: $TOKEN
write_relabel_configs:
# Filter metrics to keep
- action: keep
source_labels: [__name__]
regex: "eagle.*|\
kube_node_info.*|\
kube_node_spec_taint.*|\
container_start_time_seconds|\
container_last_seen|\
container_cpu_usage_seconds_total|\
container_fs_io_time_seconds_total|\
container_fs_write_seconds_total|\
container_fs_usage_bytes|\
container_fs_limit_bytes|\
container_memory_working_set_bytes|\
container_memory_rss|\
container_memory_usage_bytes|\
container_network_receive_bytes_total|\
container_network_transmit_bytes_total|\
machine_memory_bytes|\
machine_cpu_cores"
scrape_configs:
# Scrape config for Kubelet cAdvisor.
- job_name: 'kubernetes-cadvisor'
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- role: node
relabel_configs:
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
metric_relabel_configs:
# Only keep systemd important services like docker|containerd|kubelet and kubepods,
# Also machine_cpu_cores don't have id so we have to machine here
# The string will concat id with name and the separator is a ;
# `/;container_cpu_usage_seconds_total` OK
# `/system.slice;container_cpu_usage_seconds_total` OK
# `/system.slice/minion.service;container_cpu_usage_seconds_total` NOK, Useless
# `/kubepods/besteffort/e2514ad43202;container_cpu_usage_seconds_total` Best Effort POD OK
# `/kubepods/burstable/e2514ad43202;container_cpu_usage_seconds_total` Burstable POD OK
# `/kubepods/e2514ad43202;container_cpu_usage_seconds_total` Guaranteed POD OK
# `/docker/pod104329ff;container_cpu_usage_seconds_total` OK, Container that run on docker but not managed by kube
# `;machine_cpu_cores` OK, there is no id on these metrics, but we want to keep them also
# Only keep systemd important services like docker|containerd|kubelet and kubepods,
# Also machine_cpu_cores don't have id so we have to machine here
# The string will concat id with name and the separator is a ;
# `/;container_cpu_usage_seconds_total` OK
# `/system.slice;container_cpu_usage_seconds_total` OK
# `/system.slice/minion.service;container_cpu_usage_seconds_total` NOK, Useless
# `/kubepods/besteffort/e2514ad43202;container_cpu_usage_seconds_total` Best Effort POD OK
# `/kubepods/burstable/e2514ad43202;container_cpu_usage_seconds_total` Burstable POD OK
# `/kubepods/e2514ad43202;container_cpu_usage_seconds_total` Guaranteed POD OK
# `/docker/pod104329ff;container_cpu_usage_seconds_total` OK, Container that run on docker but not managed by kube
# `;machine_cpu_cores` OK, there is no id on these metrics, but we want to keep them also
- source_labels: [id,__name__]
regex: "^((/(system.slice(/(docker|containerd|kubelet).service)?|(kubepods|docker).*)?);.*|;(machine_cpu_cores|machine_memory_bytes))$"
action: keep
# Remove Useless parents keys like `/kubepods/burstable` or `/docker`
- source_labels: [id]
regex: "(/kubepods/burstable|/kubepods/besteffort|/kubepods|/docker)"
action: drop
# cAdvisor give metrics per container and sometimes it sum up per pod
# As we already have the child, we will sum up ourselves, so we drop metrics for the POD and keep containers metrics
# Metrics for the POD don't have container_name, so we drop if we have just the pod_name
- source_labels: [container_name,pod_name]
regex: ";(.+)"
action: drop
# Scrape config for service endpoints.
- job_name: 'kubernetes-service-endpoints'
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
action: replace
target_label: __scheme__
regex: (https?)
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
action: replace
target_label: __address__
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: namespace
- source_labels: [__meta_kubernetes_service_name]
action: replace
target_label: kubernetes_name
# Example scrape config for pods
#
# The relabeling allows the actual pod scrape endpoint to be configured via the
# following annotations:
#
# * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
# * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
# * `prometheus.io/port`: Scrape the pod on the indicated port instead of the
# pod's declared ports (default is a port-free target if none are declared).
- job_name: 'kubernetes-pods'
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
target_label: __address__
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: pod_name
- source_labels: [__meta_kubernetes_pod_node_name]
action: replace
target_label: host
- action: labeldrop
regex: (pod_template_generation|job|release|controller_revision_hash|workload_user_cattle_io_workloadselector|pod_template_hash)
alertmanager:
enabled: false
pushgateway:
enabled: false
nodeExporter:
enabled: false
server:
ingress:
enabled: true
annotations:
kubernetes.io/ingress.class: traefik
ingress.kubernetes.io/auth-type: basic
ingress.kubernetes.io/auth-secret: basic-auth
hosts:
- prometheus.domain.com
image:
tag: v2.7.1
persistentVolume:
enabled: false
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment