Prometheus ConfigMap for Openshift or Grafana

prometheus-config.yml

apiVersion: v1
data:
  prometheus.rules: |
    groups:
    - name: custom-rules
      rules:
      - alert: DBsDown
        expr: count((up{job="db-exporter"} == 0 )) > 0
        for: 2m
        labels:
          severity: critical
          service: database
        annotations:
          description: Some of the db nodes are down
      - alert: Node Down
        expr: count((up{job="node-exporter"} == 0 )) > 0
        for: 2m
        labels:
          severity: critical
          service: node
        annotations:
          description: Some of the nodes are down
      - alert: Pod is Failed 
        expr: sum(container_tasks_state{container_name!="POD",pod_name!="",image!="",id!="",state!="stopped"}) > 0
        for: 1m
        labels:
          severity: critical
          service: pod
        annotations:
          description: The Kubelet pod has been failed
          summary: Node status is Failed
  prometheus.yml: >-
    global:
      scrape_interval:     30s # Default is every 1 minute.
      evaluation_interval: 60s # The default is every 1 minute.
      
    # Alertmanager configuration

    alerting:
      alertmanagers:
      - static_configs:
        - targets:
           - alertmanager:9093

    rule_files:
      - "prometheus.rules"

    scrape_configs:
      # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
    - job_name: 'node-exporter'
        # metrics_path defaults to '/metrics'
        # scheme defaults to 'http'.
      static_configs:
        - targets: ['node1:9100', 'node2:9100', 'node3:9100', 'node4:9100']
    - job_name: 'db-exporter'
        # metrics_path defaults to '/metrics'
        # scheme defaults to 'http'.
      static_configs:
        - targets: ['db-exporter:9104']
    - job_name: 'self'
        # metrics_path defaults to '/metrics'
        # scheme defaults to 'http'.
      static_configs:
        - targets: ['localhost:9090']
    - job_name: 'cadvisor'
        # metrics_path default to '/metric'
        # scheme defaults to 'http'
      static_configs:
        - targets: ['cadvisor:8080']
    - job_name: 'kubernetes-apiservers'

      kubernetes_sd_configs:
      - role: endpoints

      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

      # Keep only the default/kubernetes service endpoints for the https port. This
      # will add targets for each API server which Kubernetes adds an endpoint to
      # the default/kubernetes service.
      relabel_configs:
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
        action: keep
        regex: default;kubernetes;https

    # Scrape config for nodes.

    #

    # Each node exposes a /metrics endpoint that contains operational metrics
    for

    # the Kubelet and other components.


    - job_name: 'kubernetes-nodes'

      kubernetes_sd_configs:
      - role: node

      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)

    # Scrape config for service endpoints.

    #

    # The relabeling allows the actual service scrape endpoint to be configured

    # via the following annotations:

    #

    # * `prometheus.io/scrape`: Only scrape services that have a value of `true`

    # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will
    need

    # to set this to `https` & most likely set the `tls_config` of the scrape
    config.

    # * `prometheus.io/path`: If the metrics path is not `/metrics` override
    this.

    # * `prometheus.io/port`: If the metrics are exposed on a different port to
    the

    # service then set this appropriately.


    - job_name: 'kubernetes-service-endpoints'

      kubernetes_sd_configs:
      - role: endpoints

    # @BEG VI: Determine if needed and how to use

    #  scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    #VI added:    ca_file:
    /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt

    #    # TODO: this should be per target
        insecure_skip_verify: true
    # @END VI: Determine if needed and how to use

      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: (.+)(?::\d+);(\d+)
        replacement: $1:$2
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: __address__
        regex: (.+)(?::\d+);(\d+)
        replacement: $1:$2
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: kubernetes_name

      # VI: merged from a sample file

      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_username]
        action: replace
        target_label: __basic_auth_username__
        regex: (.+)
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_password]
        action: replace
        target_label: __basic_auth_password__
        regex: (.+)

    # Example scrape config for pods

    #

    # The relabeling allows the actual pod scrape endpoint to be configured via
    the

    # following annotations:

    #

    # * `prometheus.io/scrape`: Only scrape pods that have a value of `true`

    # * `prometheus.io/path`: If the metrics path is not `/metrics` override
    this.

    # * `prometheus.io/port`: Scrape the pod on the indicated port instead of
    the default of `9102`.


    - job_name: 'kubernetes-pods'

      kubernetes_sd_configs:
      - role: pod

    # @BEG VI added
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        #insecure_skip_verify: true
    # @END VI added

      relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: (.+):(?:\d+);(\d+)
        replacement: ${1}:${2}
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: kubernetes_pod_name

    - job_name: 'kubernetes-services'

      metrics_path: /probe
      params:
        module: [http_2xx]

      kubernetes_sd_configs:
      - role: service

      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
        action: keep
        regex: true
      - source_labels: [__address__]
        target_label: __param_target
      - target_label: __address__
        replacement: blackbox
      - source_labels: [__param_target]
        target_label: instance
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_service_namespace]
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        target_label: kubernetes_name           
kind: ConfigMap
metadata:
  creationTimestamp: '2017-12-19T10:03:45Z'
  name: prometheus-config
  namespace: prometheus
  resourceVersion: '29769'
  selfLink: /api/v1/namespaces/prometheus/configmaps/prometheus-config
  uid: e67b9000-e4a3-11e7-8498-9e0dff7a2c22