iamabhishek-dubey/prom.yml

## prom.yml
global:
  scrape_interval:     30s # Default is every 1 minute.
  evaluation_interval: 60s # The default is every 1 minute.
  scrape_timeout: 10s      # global default is 10s

  # Attach these labels to any time series or alerts when communicating with
  # external systems (federation, remote storage, Alertmanager).
  external_labels:
      monitor: 'openshift.gcp-c.tst'

rule_files:
  - 'prometheus.rules'

# A scrape configuration for running Prometheus on a Kubernetes cluster.
# This uses separate scrape configs for cluster components (i.e. API server, node)
# and services to allow each to use different authentication configs.
#
# Kubernetes labels will be added as Prometheus labels on metrics via the
# `labelmap` relabeling action.

scrape_configs:

# scrape config for Prometheus itself.
- job_name: 'prometheus'
  metrics_path: '/metrics'
  scheme: http
  static_configs:
    - targets: ['localhost:9090']

# Scrape config for API servers.
#
# Kubernetes exposes API servers as endpoints to the default/kubernetes
# service so this uses `endpoints` role and uses relabelling to only keep
# the endpoints associated with the default/kubernetes service using the
# default named port `https`. This works for single API server deployments as
# well as HA API server deployments.

- job_name: 'kubernetes-apiservers'

  kubernetes_sd_configs:
  - role: endpoints

  scheme: https
  tls_config:
    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

  # Keep only the default/kubernetes service endpoints for the https port. This
  # will add targets for each API server which Kubernetes adds an endpoint to
  # the default/kubernetes service.
  relabel_configs:
  - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
    action: keep
    regex: default;kubernetes;https

# Scrape config for nodes.
#
# Each node exposes a /metrics endpoint that contains operational metrics for
# the Kubelet and other components.

- job_name: 'kubernetes-nodes'

  kubernetes_sd_configs:
  - role: node

  scheme: https
  tls_config:
    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

  relabel_configs:
  - action: labelmap
    regex: __meta_kubernetes_node_label_(.+)

# Scrape config for service endpoints.
#
# The relabeling allows the actual service scrape endpoint to be configured
# via the following annotations:
#
# * `prometheus.io/scrape`: Only scrape services that have a value of `true`
# * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need
# to set this to `https` & most likely set the `tls_config` of the scrape config.
# * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
# * `prometheus.io/port`: If the metrics are exposed on a different port to the
# service then set this appropriately.

- job_name: 'kubernetes-service-endpoints'

  kubernetes_sd_configs:
  - role: endpoints

# @BEG VI: Determine if needed and how to use
#  scheme: https
  tls_config:
    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
#VI added:    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
#    # TODO: this should be per target
    insecure_skip_verify: true
# @END VI: Determine if needed and how to use

  relabel_configs:
  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
    action: keep
    regex: true
  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
    action: replace
    target_label: __scheme__
    regex: (https?)
  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
    action: replace
    target_label: __metrics_path__
    regex: (.+)
  - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
    action: replace
    target_label: __address__
    regex: (.+)(?::\d+);(\d+)
    replacement: $1:$2
  - action: labelmap
    regex: __meta_kubernetes_service_label_(.+)
  - source_labels: [__meta_kubernetes_namespace]
    action: replace
    target_label: __address__
    regex: (.+)(?::\d+);(\d+)
    replacement: $1:$2
  - action: labelmap
    regex: __meta_kubernetes_service_label_(.+)
  - source_labels: [__meta_kubernetes_namespace]
    action: replace
    target_label: kubernetes_namespace
  - source_labels: [__meta_kubernetes_service_name]
    action: replace
    target_label: kubernetes_name

  # VI: merged from a sample file

  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_username]
    action: replace
    target_label: __basic_auth_username__
    regex: (.+)
  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_password]
    action: replace
    target_label: __basic_auth_password__
    regex: (.+)

# Example scrape config for pods
#
# The relabeling allows the actual pod scrape endpoint to be configured via the
# following annotations:
#
# * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
# * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
# * `prometheus.io/port`: Scrape the pod on the indicated port instead of the default of `9102`.

- job_name: 'kubernetes-pods'

  kubernetes_sd_configs:
  - role: pod

# @BEG VI added
  scheme: https
  tls_config:
    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    #insecure_skip_verify: true
# @END VI added

  relabel_configs:
  - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
    action: keep
    regex: true
  - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
    action: replace
    target_label: __metrics_path__
    regex: (.+)
  - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
    action: replace
    regex: (.+):(?:\d+);(\d+)
    replacement: ${1}:${2}
    target_label: __address__
  - action: labelmap
    regex: __meta_kubernetes_pod_label_(.+)
  - source_labels: [__meta_kubernetes_namespace]
    action: replace
    target_label: kubernetes_namespace
  - source_labels: [__meta_kubernetes_pod_name]
    action: replace
    target_label: kubernetes_pod_name

# Example scrape config for probing services via the Blackbox Exporter.
#
# The relabeling allows the actual service scrape endpoint to be configured
# via the following annotations:
#
# * `prometheus.io/probe`: Only probe services that have a value of `true`

- job_name: 'kubernetes-services'

  metrics_path: /probe
  params:
    module: [http_2xx]

  kubernetes_sd_configs:
  - role: service

  relabel_configs:
  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
    action: keep
    regex: true
  - source_labels: [__address__]
    target_label: __param_target
  - target_label: __address__
    replacement: blackbox
  - source_labels: [__param_target]
    target_label: instance
  - action: labelmap
    regex: __meta_kubernetes_service_label_(.+)
  - source_labels: [__meta_kubernetes_service_namespace]
    target_label: kubernetes_namespace
  - source_labels: [__meta_kubernetes_service_name]
    target_label: kubernetes_name
# scrape config for haproxy-exporter

- job_name: 'haproxy-scrape'
  metrics_path: '/metrics'
  scheme: http
  static_configs:
    - targets: ['haproxy-exporter:9101']

# VI: scrape config for node-exporter (leave empty line!)

- job_name: 'node-scrape'
  metrics_path: '/metrics'
  scheme: http
  static_configs:
    - targets: ['node-exporter:9100']

alerting:
  alertmanagers:
  - scheme: http
    static_configs:
    - targets:
      - "localhost:9093"
	global:
	scrape_interval: 30s # Default is every 1 minute.
	evaluation_interval: 60s # The default is every 1 minute.
	scrape_timeout: 10s # global default is 10s

	# Attach these labels to any time series or alerts when communicating with
	# external systems (federation, remote storage, Alertmanager).
	external_labels:
	monitor: 'openshift.gcp-c.tst'

	rule_files:
	- 'prometheus.rules'

	# A scrape configuration for running Prometheus on a Kubernetes cluster.
	# This uses separate scrape configs for cluster components (i.e. API server, node)
	# and services to allow each to use different authentication configs.
	#
	# Kubernetes labels will be added as Prometheus labels on metrics via the
	# `labelmap` relabeling action.

	scrape_configs:

	# scrape config for Prometheus itself.
	- job_name: 'prometheus'
	metrics_path: '/metrics'
	scheme: http
	static_configs:
	- targets: ['localhost:9090']

	# Scrape config for API servers.
	#
	# Kubernetes exposes API servers as endpoints to the default/kubernetes
	# service so this uses `endpoints` role and uses relabelling to only keep
	# the endpoints associated with the default/kubernetes service using the
	# default named port `https`. This works for single API server deployments as
	# well as HA API server deployments.

	- job_name: 'kubernetes-apiservers'

	kubernetes_sd_configs:
	- role: endpoints

	scheme: https
	tls_config:
	ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
	bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

	# Keep only the default/kubernetes service endpoints for the https port. This
	# will add targets for each API server which Kubernetes adds an endpoint to
	# the default/kubernetes service.
	relabel_configs:
	- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
	action: keep
	regex: default;kubernetes;https

	# Scrape config for nodes.
	#
	# Each node exposes a /metrics endpoint that contains operational metrics for
	# the Kubelet and other components.

	- job_name: 'kubernetes-nodes'

	kubernetes_sd_configs:
	- role: node

	scheme: https
	tls_config:
	ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
	bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

	relabel_configs:
	- action: labelmap
	regex: __meta_kubernetes_node_label_(.+)

	# Scrape config for service endpoints.
	#
	# The relabeling allows the actual service scrape endpoint to be configured
	# via the following annotations:
	#
	# * `prometheus.io/scrape`: Only scrape services that have a value of `true`
	# * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need
	# to set this to `https` & most likely set the `tls_config` of the scrape config.
	# * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
	# * `prometheus.io/port`: If the metrics are exposed on a different port to the
	# service then set this appropriately.

	- job_name: 'kubernetes-service-endpoints'

	kubernetes_sd_configs:
	- role: endpoints

	# @BEG VI: Determine if needed and how to use
	# scheme: https
	tls_config:
	ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
	#VI added: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
	# # TODO: this should be per target
	insecure_skip_verify: true
	# @END VI: Determine if needed and how to use

	relabel_configs:
	- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
	action: keep
	regex: true
	- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
	action: replace
	target_label: __scheme__
	regex: (https?)
	- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
	action: replace
	target_label: __metrics_path__
	regex: (.+)
	- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
	action: replace
	target_label: __address__
	regex: (.+)(?::\d+);(\d+)
	replacement: $1:$2
	- action: labelmap
	regex: __meta_kubernetes_service_label_(.+)
	- source_labels: [__meta_kubernetes_namespace]
	action: replace
	target_label: __address__
	regex: (.+)(?::\d+);(\d+)
	replacement: $1:$2
	- action: labelmap
	regex: __meta_kubernetes_service_label_(.+)
	- source_labels: [__meta_kubernetes_namespace]
	action: replace
	target_label: kubernetes_namespace
	- source_labels: [__meta_kubernetes_service_name]
	action: replace
	target_label: kubernetes_name

	# VI: merged from a sample file

	- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_username]
	action: replace
	target_label: __basic_auth_username__
	regex: (.+)
	- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_password]
	action: replace
	target_label: __basic_auth_password__
	regex: (.+)

	# Example scrape config for pods
	#
	# The relabeling allows the actual pod scrape endpoint to be configured via the
	# following annotations:
	#
	# * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
	# * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
	# * `prometheus.io/port`: Scrape the pod on the indicated port instead of the default of `9102`.

	- job_name: 'kubernetes-pods'

	kubernetes_sd_configs:
	- role: pod

	# @BEG VI added
	scheme: https
	tls_config:
	ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
	#insecure_skip_verify: true
	# @END VI added

	relabel_configs:
	- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
	action: keep
	regex: true
	- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
	action: replace
	target_label: __metrics_path__
	regex: (.+)
	- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
	action: replace
	regex: (.+):(?:\d+);(\d+)
	replacement: ${1}:${2}
	target_label: __address__
	- action: labelmap
	regex: __meta_kubernetes_pod_label_(.+)
	- source_labels: [__meta_kubernetes_namespace]
	action: replace
	target_label: kubernetes_namespace
	- source_labels: [__meta_kubernetes_pod_name]
	action: replace
	target_label: kubernetes_pod_name

	# Example scrape config for probing services via the Blackbox Exporter.
	#
	# The relabeling allows the actual service scrape endpoint to be configured
	# via the following annotations:
	#
	# * `prometheus.io/probe`: Only probe services that have a value of `true`

	- job_name: 'kubernetes-services'

	metrics_path: /probe
	params:
	module: [http_2xx]

	kubernetes_sd_configs:
	- role: service

	relabel_configs:
	- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
	action: keep
	regex: true
	- source_labels: [__address__]
	target_label: __param_target
	- target_label: __address__
	replacement: blackbox
	- source_labels: [__param_target]
	target_label: instance
	- action: labelmap
	regex: __meta_kubernetes_service_label_(.+)
	- source_labels: [__meta_kubernetes_service_namespace]
	target_label: kubernetes_namespace
	- source_labels: [__meta_kubernetes_service_name]
	target_label: kubernetes_name
	# scrape config for haproxy-exporter

	- job_name: 'haproxy-scrape'
	metrics_path: '/metrics'
	scheme: http
	static_configs:
	- targets: ['haproxy-exporter:9101']

	# VI: scrape config for node-exporter (leave empty line!)

	- job_name: 'node-scrape'
	metrics_path: '/metrics'
	scheme: http
	static_configs:
	- targets: ['node-exporter:9100']

	alerting:
	alertmanagers:
	- scheme: http
	static_configs:
	- targets:
	- "localhost:9093"