Created
February 18, 2019 14:56
-
-
Save karthikeayan/41ab3dc4ed0c344bbab89ebcb1d33d16 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
global: | |
scrape_interval: 1m | |
scrape_timeout: 10s | |
evaluation_interval: 1m | |
alerting: | |
alertmanagers: | |
- kubernetes_sd_configs: | |
- role: pod | |
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token | |
tls_config: | |
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt | |
insecure_skip_verify: false | |
scheme: http | |
timeout: 10s | |
relabel_configs: | |
- source_labels: [__meta_kubernetes_namespace] | |
separator: ; | |
regex: prometheus | |
replacement: $1 | |
action: keep | |
- source_labels: [__meta_kubernetes_pod_label_app] | |
separator: ; | |
regex: prometheus | |
replacement: $1 | |
action: keep | |
- source_labels: [__meta_kubernetes_pod_label_component] | |
separator: ; | |
regex: alertmanager | |
replacement: $1 | |
action: keep | |
- source_labels: [__meta_kubernetes_pod_container_port_number] | |
separator: ; | |
regex: null | |
replacement: $1 | |
action: drop | |
rule_files: | |
- /etc/config/rules | |
- /etc/config/alerts | |
scrape_configs: | |
- job_name: prometheus | |
scrape_interval: 1m | |
scrape_timeout: 10s | |
metrics_path: /metrics | |
scheme: http | |
static_configs: | |
- targets: | |
- localhost:9090 | |
- job_name: kubernetes-apiservers | |
scrape_interval: 1m | |
scrape_timeout: 10s | |
metrics_path: /metrics | |
scheme: https | |
kubernetes_sd_configs: | |
- role: endpoints | |
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token | |
tls_config: | |
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt | |
insecure_skip_verify: true | |
relabel_configs: | |
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] | |
separator: ; | |
regex: default;kubernetes;https | |
replacement: $1 | |
action: keep | |
- job_name: kubernetes-nodes | |
scrape_interval: 1m | |
scrape_timeout: 10s | |
metrics_path: /metrics | |
scheme: https | |
kubernetes_sd_configs: | |
- role: node | |
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token | |
tls_config: | |
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt | |
insecure_skip_verify: true | |
relabel_configs: | |
- separator: ; | |
regex: __meta_kubernetes_node_label_(.+) | |
replacement: $1 | |
action: labelmap | |
- separator: ; | |
regex: (.*) | |
target_label: __address__ | |
replacement: kubernetes.default.svc:443 | |
action: replace | |
- source_labels: [__meta_kubernetes_node_name] | |
separator: ; | |
regex: (.+) | |
target_label: __metrics_path__ | |
replacement: /api/v1/nodes/$1/proxy/metrics | |
action: replace | |
- job_name: kubernetes-nodes-cadvisor | |
scrape_interval: 1m | |
scrape_timeout: 10s | |
metrics_path: /metrics | |
scheme: https | |
kubernetes_sd_configs: | |
- role: node | |
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token | |
tls_config: | |
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt | |
insecure_skip_verify: true | |
relabel_configs: | |
- separator: ; | |
regex: __meta_kubernetes_node_label_(.+) | |
replacement: $1 | |
action: labelmap | |
- separator: ; | |
regex: (.*) | |
target_label: __address__ | |
replacement: kubernetes.default.svc:443 | |
action: replace | |
- source_labels: [__meta_kubernetes_node_name] | |
separator: ; | |
regex: (.+) | |
target_label: __metrics_path__ | |
replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor | |
action: replace | |
- job_name: kubernetes-service-endpoints | |
scrape_interval: 1m | |
scrape_timeout: 10s | |
metrics_path: /metrics | |
scheme: http | |
kubernetes_sd_configs: | |
- role: endpoints | |
relabel_configs: | |
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] | |
separator: ; | |
regex: "true" | |
replacement: $1 | |
action: keep | |
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] | |
separator: ; | |
regex: (https?) | |
target_label: __scheme__ | |
replacement: $1 | |
action: replace | |
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] | |
separator: ; | |
regex: (.+) | |
target_label: __metrics_path__ | |
replacement: $1 | |
action: replace | |
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] | |
separator: ; | |
regex: ([^:]+)(?::\d+)?;(\d+) | |
target_label: __address__ | |
replacement: $1:$2 | |
action: replace | |
- separator: ; | |
regex: __meta_kubernetes_service_label_(.+) | |
replacement: $1 | |
action: labelmap | |
- source_labels: [__meta_kubernetes_namespace] | |
separator: ; | |
regex: (.*) | |
target_label: kubernetes_namespace | |
replacement: $1 | |
action: replace | |
- source_labels: [__meta_kubernetes_service_name] | |
separator: ; | |
regex: (.*) | |
target_label: kubernetes_name | |
replacement: $1 | |
action: replace | |
- source_labels: [__meta_kubernetes_pod_node_name] | |
separator: ; | |
regex: (.*) | |
target_label: kubernetes_node | |
replacement: $1 | |
action: replace | |
- job_name: prometheus-pushgateway | |
honor_labels: true | |
scrape_interval: 1m | |
scrape_timeout: 10s | |
metrics_path: /metrics | |
scheme: http | |
kubernetes_sd_configs: | |
- role: service | |
relabel_configs: | |
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe] | |
separator: ; | |
regex: pushgateway | |
replacement: $1 | |
action: keep | |
- job_name: kubernetes-services | |
params: | |
module: | |
- http_2xx | |
scrape_interval: 1m | |
scrape_timeout: 10s | |
metrics_path: /probe | |
scheme: http | |
kubernetes_sd_configs: | |
- role: service | |
relabel_configs: | |
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe] | |
separator: ; | |
regex: "true" | |
replacement: $1 | |
action: keep | |
- source_labels: [__address__] | |
separator: ; | |
regex: (.*) | |
target_label: __param_target | |
replacement: $1 | |
action: replace | |
- separator: ; | |
regex: (.*) | |
target_label: __address__ | |
replacement: blackbox | |
action: replace | |
- source_labels: [__param_target] | |
separator: ; | |
regex: (.*) | |
target_label: instance | |
replacement: $1 | |
action: replace | |
- separator: ; | |
regex: __meta_kubernetes_service_label_(.+) | |
replacement: $1 | |
action: labelmap | |
- source_labels: [__meta_kubernetes_namespace] | |
separator: ; | |
regex: (.*) | |
target_label: kubernetes_namespace | |
replacement: $1 | |
action: replace | |
- source_labels: [__meta_kubernetes_service_name] | |
separator: ; | |
regex: (.*) | |
target_label: kubernetes_name | |
replacement: $1 | |
action: replace | |
- job_name: kubernetes-pods | |
scrape_interval: 1m | |
scrape_timeout: 10s | |
metrics_path: /metrics | |
scheme: http | |
kubernetes_sd_configs: | |
- role: pod | |
relabel_configs: | |
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] | |
separator: ; | |
regex: "true" | |
replacement: $1 | |
action: keep | |
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] | |
separator: ; | |
regex: (.+) | |
target_label: __metrics_path__ | |
replacement: $1 | |
action: replace | |
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] | |
separator: ; | |
regex: ([^:]+)(?::\d+)?;(\d+) | |
target_label: __address__ | |
replacement: $1:$2 | |
action: replace | |
- separator: ; | |
regex: __meta_kubernetes_pod_label_(.+) | |
replacement: $1 | |
action: labelmap | |
- source_labels: [__meta_kubernetes_namespace] | |
separator: ; | |
regex: (.*) | |
target_label: kubernetes_namespace | |
replacement: $1 | |
action: replace | |
- source_labels: [__meta_kubernetes_pod_name] | |
separator: ; | |
regex: (.*) | |
target_label: kubernetes_pod_name | |
replacement: $1 | |
action: replace |
Yes, did you try adding a label "prometheus.io/scrape: true" to the pods?
By default installed Prometheus with node exporters shows in Grafana only node's metrics and my pods' metrics - no.
Do you mean if i just add "prometheus.io/scrape: true" to my deployments the pods' metrics will appear in Grafana?
Yes, if I remember correctly thats what it is. Unfortunately I don't have access to the Kubernetes cluster to find the exact fix for you.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Hi, Have you found the solution for your post: https://stackoverflow.com/questions/54749175/prometheus-is-not-collecting-pod-metrics ?
I faced the same issue(
Thanks!