jspaleta/cilium-observability.yaml

## cilium-observability.yaml
---
apiVersion: v1
kind: Namespace
metadata:
  name: cilium-observability
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus-k8s
  namespace: cilium-observability
---
apiVersion: v1
kind: ConfigMap
metadata:
  labels:
    app: grafana
  name: grafana-config
  namespace: cilium-observability
data:
  config.yaml: |
    apiVersion: 1
    providers:
    - name: 'cilium-dashboard-config'
      orgId: 1
      folder: ''
      type: file
      disableDeletion: true
      options:
        path: '/configmap/dashboards/'
  grafana.ini: |
    app_mode = production
    [paths]
    # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)
    data = data
    # Temporary files in `data` directory older than given duration will be removed
    temp_data_lifetime = 24h
    # Directory where grafana can store logs
    logs = data/log
    # Directory where grafana will automatically scan and look for plugins
    plugins = data/plugins
    # folder that contains provisioning config files that grafana will apply on startup and while running.
    provisioning = /configmap/grafana/provisioning
    #################################### Server ##############################
    [server]
    # Protocol (http, https, socket)
    protocol = http
    # The http port to use
    http_port = 3000
    # enable gzip
    enable_gzip = false
    #################################### Session #############################
    [session]
    # Either "memory", "file", "redis", "mysql", "postgres", "memcache", default is "file"
    provider = file

    provider_config = sessions

    # Session cookie name
    cookie_name = grafana_sess

    # If you use session in https only, default is false
    cookie_secure = true

    # Session life time, default is 86400
    session_life_time = 86400
    gc_interval_time = 86400

    # Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours)
    conn_max_lifetime = 14400

    #################################### Data proxy ###########################
    [dataproxy]

    # This enables data proxy logging, default is false
    logging = false

    # How long the data proxy should wait before timing out default is 30 (seconds)
    timeout = 30

    #################################### Analytics ###########################
    [analytics]
    # Server reporting, sends usage counters to stats.grafana.org every 24 hours.
    # No ip addresses are being tracked, only simple counters to track
    # running instances, dashboard and error counts. It is very helpful to us.
    # Change this option to false to disable reporting.
    reporting_enabled = false

    # Set to false to disable all checks to https://grafana.com
    # for new versions (grafana itself and plugins), check is used
    # in some UI views to notify that grafana or plugin update exists
    # This option does not cause any auto updates, nor send any information
    # only a GET request to https://grafana.com to get latest versions
    check_for_updates = false

    #################################### Security ############################
    [security]
    # default admin user, created on startup
    admin_user = admin

    # default admin password, can be changed before first start of grafana, or in profile settings
    admin_password = admin

    # used for signing
    secret_key = SW2YcwTIb9zpOOhoPsMm

    # disable gravatar profile images
    disable_gravatar = true

    # disable protection against brute force login attempts
    disable_brute_force_login_protection = false

    # set to true if you host Grafana behind HTTPS. default is false.
    cookie_secure = true

    # set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict" and "none"
    cookie_samesite = lax

    #################################### Dashboards ##################

    [dashboards]
    # Number dashboard versions to keep (per dashboard). Default: 20, Minimum: 1
    versions_to_keep = 20

    #################################### Users ###############################
    [users]
    # disable user signup / registration
    allow_sign_up = false

    # Allow non admin users to create organizations
    allow_org_create = false

    # Set to true to automatically assign new users to the default organization (id 1)
    auto_assign_org = true

    # Set this value to automatically add new users to the provided organization (if auto_assign_org above is set to true)
    auto_assign_org_id = 1

    # Default role new users will be automatically assigned (if auto_assign_org above is set to true)
    auto_assign_org_role = Viewer

    # Require email validation before sign up completes
    verify_email_enabled = false

    # Background text for the user field on the login page
    login_hint = email or username

    # Default UI theme ("dark" or "light")
    default_theme = dark

    # External user management
    external_manage_link_url =
    external_manage_link_name =
    external_manage_info =

    # Viewers can edit/inspect dashboard settings in the browser. But not save the dashboard.
    viewers_can_edit = false

    [auth]
    # Login cookie name
    login_cookie_name = grafana_session

    # The lifetime (days) an authenticated user can be inactive before being required to login at next visit. Default is 7 days.
    login_maximum_inactive_lifetime_days = 7

    # The maximum lifetime (days) an authenticated user can be logged in since login time before being required to login. Default is 30 days.
    login_maximum_lifetime_days = 30

    # How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes.
    token_rotation_interval_minutes = 10

    # Set to true to disable (hide) the login form, useful if you use OAuth
    disable_login_form = false

    # Set to true to disable the signout link in the side menu. useful if you use auth.proxy
    disable_signout_menu = false

    # URL to redirect the user to after sign out
    signout_redirect_url =

    # Set to true to attempt login with OAuth automatically, skipping the login screen.
    # This setting is ignored if multiple OAuth providers are configured.
    oauth_auto_login = false

    #################################### Anonymous Auth ######################
    [auth.anonymous]
    # enable anonymous access
    enabled = true

    # specify organization name that should be used for unauthenticated users
    org_name = Main Org.

    # specify role for unauthenticated users
    org_role = Admin


    #################################### Basic Auth ##########################
    [auth.basic]
    enabled = false

    #################################### Logging ##########################
    [log]
    # Either "console", "file", "syslog". Default is console and file
    # Use space to separate multiple modes, e.g. "console file"
    mode = console file

    # Either "debug", "info", "warn", "error", "critical", default is "info"
    level = info

    # optional settings to set different levels for specific loggers. Ex filters = sqlstore:debug
    filters =

    # For "console" mode only
    [log.console]
    level =

    # log line format, valid options are text, console and json
    format = console

    # For "file" mode only
    [log.file]
    level =

    # log line format, valid options are text, console and json
    format = text

    # This enables automated log rotate(switch of following options), default is true
    log_rotate = true

    # Max line number of single file, default is 1000000
    max_lines = 1000000

    # Max size shift of single file, default is 28 means 1 << 28, 256MB
    max_size_shift = 28

    # Segment log daily, default is true
    daily_rotate = true

    # Expired days of log file(delete after max days), default is 7
    max_days = 7

    [log.syslog]
    level =

    # log line format, valid options are text, console and json
    format = text

    # Syslog network type and address. This can be udp, tcp, or unix. If left blank, the default unix endpoints will be used.
    network =
    address =

    # Syslog facility. user, daemon and local0 through local7 are valid.
    facility =

    # Syslog tag. By default, the process' argv[0] is used.
    tag =

    #################################### Usage Quotas ########################
    [quota]
    enabled = false

    #### set quotas to -1 to make unlimited. ####
    # limit number of users per Org.
    org_user = 10

    # limit number of dashboards per Org.
    org_dashboard = 100

    # limit number of data_sources per Org.
    org_data_source = 10

    # limit number of api_keys per Org.
    org_api_key = 10

    # limit number of orgs a user can create.
    user_org = 10

    # Global limit of users.
    global_user = -1

    # global limit of orgs.
    global_org = -1

    # global limit of dashboards
    global_dashboard = -1

    # global limit of api_keys
    global_api_key = -1

    # global limit on number of logged in users.
    global_session = -1

    #################################### Alerting ############################
    [alerting]
    # Disable alerting engine & UI features
    enabled = true
    # Makes it possible to turn off alert rule execution but alerting UI is visible
    execute_alerts = true

    # Default setting for new alert rules. Defaults to categorize error and timeouts as alerting. (alerting, keep_state)
    error_or_timeout = alerting

    # Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok)
    nodata_or_nullvalues = no_data

    # Alert notifications can include images, but rendering many images at the same time can overload the server
    # This limit will protect the server from render overloading and make sure notifications are sent out quickly
    concurrent_render_limit = 5

    #################################### Explore #############################
    [explore]
    # Enable the Explore section
    enabled = true

    #################################### Internal Grafana Metrics ############
    # Metrics available at HTTP API Url /metrics
    [metrics]
    enabled           = true
    interval_seconds  = 10

    #If both are set, basic auth will be required for the metrics endpoint.
    basic_auth_username =
    basic_auth_password =
  prometheus-datasource.yaml: |-
    apiVersion: 1
    deleteDatasources:
      - name: prometheus
        orgId: 1
    datasources:
    - name: prometheus
      type: prometheus
      access: proxy
      orgId: 1
      url: http://prometheus:9090
      basicAuth: false
      editable: true
---
kind: ConfigMap
metadata:
  name: prometheus
  namespace: cilium-observability
apiVersion: v1
data:
  prometheus.yaml: |
    global:
      scrape_interval: 10s
      scrape_timeout: 10s
      evaluation_interval: 10s
    rule_files:
      - "/etc/prometheus-rules/*.rules"
    scrape_configs:
      # https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml#L79
      - job_name: 'kubernetes-endpoints'
        kubernetes_sd_configs:
          - role: endpoints
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_label_k8s_app]
            action: keep
            regex: cilium
          - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
            action: keep
            regex: true
          - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
            action: replace
            target_label: __scheme__
            regex: (https?)
          - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
            action: replace
            target_label: __metrics_path__
            regex: (.+)
          - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
            action: replace
            target_label: __address__
            regex: (.+)(?::\d+);(\d+)
            replacement: $1:$2
          - action: labelmap
            regex: __meta_kubernetes_service_label_(.+)
          - source_labels: [__meta_kubernetes_namespace]
            action: replace
            target_label: namespace
          - source_labels: [__meta_kubernetes_service_name]
            action: replace
            target_label: service

      # https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml#L156
      - job_name: 'kubernetes-pods'
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
            action: keep
            regex: true
          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
            action: replace
            target_label: __metrics_path__
            regex: (.+)
          - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
            action: replace
            regex: (.+):(?:\d+);(\d+)
            replacement: ${1}:${2}
            target_label: __address__
          - action: labelmap
            regex: __meta_kubernetes_pod_label_(.+)
          - source_labels: [__meta_kubernetes_namespace]
            action: replace
            target_label: namespace
          - source_labels: [__meta_kubernetes_pod_name]
            action: replace
            target_label: pod
          - source_labels: [__meta_kubernetes_pod_container_port_number]
            action: keep
            regex: \d+

      # https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml#L119
      - job_name: 'kubernetes-services'
        metrics_path: /metrics
        params:
          module: [http_2xx]
        kubernetes_sd_configs:
          - role: service
        relabel_configs:
          - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
            action: keep
            regex: true
          - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
            action: replace
            regex: ([^:]+)(?::\d+)?;(\d+)
            replacement: ${1}:${2}
            target_label: __address__
          - action: labelmap
            regex: __meta_kubernetes_service_label_(.+)
          - source_labels: [__meta_kubernetes_namespace]
            target_label: namespace
          - source_labels: [__meta_kubernetes_service_name]
            target_label: service

      - job_name: 'kubernetes-cadvisor'
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        kubernetes_sd_configs:
          - role: node
        relabel_configs:
          - action: labelmap
            regex: __meta_kubernetes_node_label_(.+)
          - target_label: __address__
            replacement: kubernetes.default.svc:443
          - source_labels: [__meta_kubernetes_node_name]
            regex: (.+)
            target_label: __metrics_path__
            replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prometheus
rules:
- apiGroups:
  - ""
  resources:
  - nodes
  - nodes/proxy
  - services
  - endpoints
  - pods
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - configmaps
  verbs:
  - get
- nonResourceURLs:
  - /metrics
  verbs:
  - get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus
subjects:
- kind: ServiceAccount
  name: prometheus-k8s
  namespace: cilium-observability
---
# Source: prometheus/templates/grafana.yaml
apiVersion: v1
kind: Service
metadata:
  labels:
    app: grafana
  name: grafana
  namespace: cilium-observability
spec:
  ports:
    - port: 3000
      protocol: TCP
      targetPort: 3000
  selector:
    app: grafana
  type: ClusterIP
---
# Source: prometheus/templates/prometheus.yaml
apiVersion: v1
kind: Service
metadata:
  labels:
    app: prometheus
  name: prometheus
  namespace: cilium-observability
spec:
  ports:
  - name: webui
    port: 9090
    protocol: TCP
    targetPort: 9090
  selector:
    app: prometheus
  type: ClusterIP
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: grafana
    component: core
  name: grafana
  namespace: cilium-observability
spec:
  replicas: 1
  selector:
    matchLabels:
      app: grafana
  template:
    metadata:
      labels:
        app: grafana
    spec:
      containers:
        - env:
            - name: GF_PATHS_CONFIG
              value: /configmap/grafana/grafana-config.ini
            - name: GF_PATHS_PROVISIONING
              value: /configmap/grafana/provisioning
          image: "docker.io/grafana/grafana:9.3.6"
          imagePullPolicy: IfNotPresent
          name: grafana-core
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /login
              port: 3000
              scheme: HTTP
          volumeMounts:
            - mountPath: /configmap/grafana
              name: grafana-config
              readOnly: true
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      volumes:
        - configMap:
            defaultMode: 420
            items:
              - key: grafana.ini
                path: grafana-config.ini
              - key: prometheus-datasource.yaml
                path: provisioning/datasources/prometheus.yaml
              - key: config.yaml
                path: provisioning/dashboards/config.yaml
            name: grafana-config
          name: grafana-config
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: prometheus
  name: prometheus
  namespace: cilium-observability
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus
  template:
    metadata:
      labels:
        app: prometheus
      name: prometheus-main
    spec:
      containers:
      - args:
        - --config.file=/etc/prometheus/prometheus.yaml
        - --storage.tsdb.path=/prometheus/
        - --log.level=info
        - --enable-feature=exemplar-storage
        image: "prom/prometheus:v2.42.0"
        imagePullPolicy: IfNotPresent
        name: prometheus
        ports:
        - containerPort: 9090
          name: webui
          protocol: TCP
        volumeMounts:
        - mountPath: /etc/prometheus
          name: config-volume
          readOnly: true
        - mountPath: /prometheus/
          name: storage
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      serviceAccount: prometheus-k8s
      volumes:
      - configMap:
          name: prometheus
        name: config-volume
      - emptyDir: {}
        name: storage


## hubble-policy-metrics-config.yaml
hubble:
  enabled: true
  relay:
    enabled: true
  ui:
    enabled: true
  metrics:
    enabled:
      - "policy:sourceContext=app|workload-name|pod|reserved-identity;destinationContext=app|workload-name|pod|dns|reserved-identity;labelsContext=source_namespace,destination_namespace"
	---
	apiVersion: v1
	kind: Namespace
	metadata:
	name: cilium-observability
	---
	apiVersion: v1
	kind: ServiceAccount
	metadata:
	name: prometheus-k8s
	namespace: cilium-observability
	---
	apiVersion: v1
	kind: ConfigMap
	metadata:
	labels:
	app: grafana
	name: grafana-config
	namespace: cilium-observability
	data:
	config.yaml: \|
	apiVersion: 1
	providers:
	- name: 'cilium-dashboard-config'
	orgId: 1
	folder: ''
	type: file
	disableDeletion: true
	options:
	path: '/configmap/dashboards/'
	grafana.ini: \|
	app_mode = production
	[paths]
	# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)
	data = data
	# Temporary files in `data` directory older than given duration will be removed
	temp_data_lifetime = 24h
	# Directory where grafana can store logs
	logs = data/log
	# Directory where grafana will automatically scan and look for plugins
	plugins = data/plugins
	# folder that contains provisioning config files that grafana will apply on startup and while running.
	provisioning = /configmap/grafana/provisioning
	#################################### Server ##############################
	[server]
	# Protocol (http, https, socket)
	protocol = http
	# The http port to use
	http_port = 3000
	# enable gzip
	enable_gzip = false
	#################################### Session #############################
	[session]
	# Either "memory", "file", "redis", "mysql", "postgres", "memcache", default is "file"
	provider = file

	provider_config = sessions

	# Session cookie name
	cookie_name = grafana_sess

	# If you use session in https only, default is false
	cookie_secure = true

	# Session life time, default is 86400
	session_life_time = 86400
	gc_interval_time = 86400

	# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours)
	conn_max_lifetime = 14400

	#################################### Data proxy ###########################
	[dataproxy]

	# This enables data proxy logging, default is false
	logging = false

	# How long the data proxy should wait before timing out default is 30 (seconds)
	timeout = 30

	#################################### Analytics ###########################
	[analytics]
	# Server reporting, sends usage counters to stats.grafana.org every 24 hours.
	# No ip addresses are being tracked, only simple counters to track
	# running instances, dashboard and error counts. It is very helpful to us.
	# Change this option to false to disable reporting.
	reporting_enabled = false

	# Set to false to disable all checks to https://grafana.com
	# for new versions (grafana itself and plugins), check is used
	# in some UI views to notify that grafana or plugin update exists
	# This option does not cause any auto updates, nor send any information
	# only a GET request to https://grafana.com to get latest versions
	check_for_updates = false

	#################################### Security ############################
	[security]
	# default admin user, created on startup
	admin_user = admin

	# default admin password, can be changed before first start of grafana, or in profile settings
	admin_password = admin

	# used for signing
	secret_key = SW2YcwTIb9zpOOhoPsMm

	# disable gravatar profile images
	disable_gravatar = true

	# disable protection against brute force login attempts
	disable_brute_force_login_protection = false

	# set to true if you host Grafana behind HTTPS. default is false.
	cookie_secure = true

	# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict" and "none"
	cookie_samesite = lax

	#################################### Dashboards ##################

	[dashboards]
	# Number dashboard versions to keep (per dashboard). Default: 20, Minimum: 1
	versions_to_keep = 20

	#################################### Users ###############################
	[users]
	# disable user signup / registration
	allow_sign_up = false

	# Allow non admin users to create organizations
	allow_org_create = false

	# Set to true to automatically assign new users to the default organization (id 1)
	auto_assign_org = true

	# Set this value to automatically add new users to the provided organization (if auto_assign_org above is set to true)
	auto_assign_org_id = 1

	# Default role new users will be automatically assigned (if auto_assign_org above is set to true)
	auto_assign_org_role = Viewer

	# Require email validation before sign up completes
	verify_email_enabled = false

	# Background text for the user field on the login page
	login_hint = email or username

	# Default UI theme ("dark" or "light")
	default_theme = dark

	# External user management
	external_manage_link_url =
	external_manage_link_name =
	external_manage_info =

	# Viewers can edit/inspect dashboard settings in the browser. But not save the dashboard.
	viewers_can_edit = false

	[auth]
	# Login cookie name
	login_cookie_name = grafana_session

	# The lifetime (days) an authenticated user can be inactive before being required to login at next visit. Default is 7 days.
	login_maximum_inactive_lifetime_days = 7

	# The maximum lifetime (days) an authenticated user can be logged in since login time before being required to login. Default is 30 days.
	login_maximum_lifetime_days = 30

	# How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes.
	token_rotation_interval_minutes = 10

	# Set to true to disable (hide) the login form, useful if you use OAuth
	disable_login_form = false

	# Set to true to disable the signout link in the side menu. useful if you use auth.proxy
	disable_signout_menu = false

	# URL to redirect the user to after sign out
	signout_redirect_url =

	# Set to true to attempt login with OAuth automatically, skipping the login screen.
	# This setting is ignored if multiple OAuth providers are configured.
	oauth_auto_login = false

	#################################### Anonymous Auth ######################
	[auth.anonymous]
	# enable anonymous access
	enabled = true

	# specify organization name that should be used for unauthenticated users
	org_name = Main Org.

	# specify role for unauthenticated users
	org_role = Admin


	#################################### Basic Auth ##########################
	[auth.basic]
	enabled = false

	#################################### Logging ##########################
	[log]
	# Either "console", "file", "syslog". Default is console and file
	# Use space to separate multiple modes, e.g. "console file"
	mode = console file

	# Either "debug", "info", "warn", "error", "critical", default is "info"
	level = info

	# optional settings to set different levels for specific loggers. Ex filters = sqlstore:debug
	filters =

	# For "console" mode only
	[log.console]
	level =

	# log line format, valid options are text, console and json
	format = console

	# For "file" mode only
	[log.file]
	level =

	# log line format, valid options are text, console and json
	format = text

	# This enables automated log rotate(switch of following options), default is true
	log_rotate = true

	# Max line number of single file, default is 1000000
	max_lines = 1000000

	# Max size shift of single file, default is 28 means 1 << 28, 256MB
	max_size_shift = 28

	# Segment log daily, default is true
	daily_rotate = true

	# Expired days of log file(delete after max days), default is 7
	max_days = 7

	[log.syslog]
	level =

	# log line format, valid options are text, console and json
	format = text

	# Syslog network type and address. This can be udp, tcp, or unix. If left blank, the default unix endpoints will be used.
	network =
	address =

	# Syslog facility. user, daemon and local0 through local7 are valid.
	facility =

	# Syslog tag. By default, the process' argv[0] is used.
	tag =

	#################################### Usage Quotas ########################
	[quota]
	enabled = false

	#### set quotas to -1 to make unlimited. ####
	# limit number of users per Org.
	org_user = 10

	# limit number of dashboards per Org.
	org_dashboard = 100

	# limit number of data_sources per Org.
	org_data_source = 10

	# limit number of api_keys per Org.
	org_api_key = 10

	# limit number of orgs a user can create.
	user_org = 10

	# Global limit of users.
	global_user = -1

	# global limit of orgs.
	global_org = -1

	# global limit of dashboards
	global_dashboard = -1

	# global limit of api_keys
	global_api_key = -1

	# global limit on number of logged in users.
	global_session = -1

	#################################### Alerting ############################
	[alerting]
	# Disable alerting engine & UI features
	enabled = true
	# Makes it possible to turn off alert rule execution but alerting UI is visible
	execute_alerts = true

	# Default setting for new alert rules. Defaults to categorize error and timeouts as alerting. (alerting, keep_state)
	error_or_timeout = alerting

	# Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok)
	nodata_or_nullvalues = no_data

	# Alert notifications can include images, but rendering many images at the same time can overload the server
	# This limit will protect the server from render overloading and make sure notifications are sent out quickly
	concurrent_render_limit = 5

	#################################### Explore #############################
	[explore]
	# Enable the Explore section
	enabled = true

	#################################### Internal Grafana Metrics ############
	# Metrics available at HTTP API Url /metrics
	[metrics]
	enabled = true
	interval_seconds = 10

	#If both are set, basic auth will be required for the metrics endpoint.
	basic_auth_username =
	basic_auth_password =
	prometheus-datasource.yaml: \|-
	apiVersion: 1
	deleteDatasources:
	- name: prometheus
	orgId: 1
	datasources:
	- name: prometheus
	type: prometheus
	access: proxy
	orgId: 1
	url: http://prometheus:9090
	basicAuth: false
	editable: true
	---
	kind: ConfigMap
	metadata:
	name: prometheus
	namespace: cilium-observability
	apiVersion: v1
	data:
	prometheus.yaml: \|
	global:
	scrape_interval: 10s
	scrape_timeout: 10s
	evaluation_interval: 10s
	rule_files:
	- "/etc/prometheus-rules/*.rules"
	scrape_configs:
	# https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml#L79
	- job_name: 'kubernetes-endpoints'
	kubernetes_sd_configs:
	- role: endpoints
	relabel_configs:
	- source_labels: [__meta_kubernetes_pod_label_k8s_app]
	action: keep
	regex: cilium
	- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
	action: keep
	regex: true
	- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
	action: replace
	target_label: __scheme__
	regex: (https?)
	- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
	action: replace
	target_label: __metrics_path__
	regex: (.+)
	- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
	action: replace
	target_label: __address__
	regex: (.+)(?::\d+);(\d+)
	replacement: $1:$2
	- action: labelmap
	regex: __meta_kubernetes_service_label_(.+)
	- source_labels: [__meta_kubernetes_namespace]
	action: replace
	target_label: namespace
	- source_labels: [__meta_kubernetes_service_name]
	action: replace
	target_label: service

	# https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml#L156
	- job_name: 'kubernetes-pods'
	kubernetes_sd_configs:
	- role: pod
	relabel_configs:
	- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
	action: keep
	regex: true
	- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
	action: replace
	target_label: __metrics_path__
	regex: (.+)
	- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
	action: replace
	regex: (.+):(?:\d+);(\d+)
	replacement: ${1}:${2}
	target_label: __address__
	- action: labelmap
	regex: __meta_kubernetes_pod_label_(.+)
	- source_labels: [__meta_kubernetes_namespace]
	action: replace
	target_label: namespace
	- source_labels: [__meta_kubernetes_pod_name]
	action: replace
	target_label: pod
	- source_labels: [__meta_kubernetes_pod_container_port_number]
	action: keep
	regex: \d+

	# https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml#L119
	- job_name: 'kubernetes-services'
	metrics_path: /metrics
	params:
	module: [http_2xx]
	kubernetes_sd_configs:
	- role: service
	relabel_configs:
	- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
	action: keep
	regex: true
	- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
	action: replace
	regex: ([^:]+)(?::\d+)?;(\d+)
	replacement: ${1}:${2}
	target_label: __address__
	- action: labelmap
	regex: __meta_kubernetes_service_label_(.+)
	- source_labels: [__meta_kubernetes_namespace]
	target_label: namespace
	- source_labels: [__meta_kubernetes_service_name]
	target_label: service

	- job_name: 'kubernetes-cadvisor'
	scheme: https
	tls_config:
	ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
	bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
	kubernetes_sd_configs:
	- role: node
	relabel_configs:
	- action: labelmap
	regex: __meta_kubernetes_node_label_(.+)
	- target_label: __address__
	replacement: kubernetes.default.svc:443
	- source_labels: [__meta_kubernetes_node_name]
	regex: (.+)
	target_label: __metrics_path__
	replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
	---
	apiVersion: rbac.authorization.k8s.io/v1
	kind: ClusterRole
	metadata:
	name: prometheus
	rules:
	- apiGroups:
	- ""
	resources:
	- nodes
	- nodes/proxy
	- services
	- endpoints
	- pods
	verbs:
	- get
	- list
	- watch
	- apiGroups:
	- ""
	resources:
	- configmaps
	verbs:
	- get
	- nonResourceURLs:
	- /metrics
	verbs:
	- get
	---
	apiVersion: rbac.authorization.k8s.io/v1
	kind: ClusterRoleBinding
	metadata:
	name: prometheus
	roleRef:
	apiGroup: rbac.authorization.k8s.io
	kind: ClusterRole
	name: prometheus
	subjects:
	- kind: ServiceAccount
	name: prometheus-k8s
	namespace: cilium-observability
	---
	# Source: prometheus/templates/grafana.yaml
	apiVersion: v1
	kind: Service
	metadata:
	labels:
	app: grafana
	name: grafana
	namespace: cilium-observability
	spec:
	ports:
	- port: 3000
	protocol: TCP
	targetPort: 3000
	selector:
	app: grafana
	type: ClusterIP
	---
	# Source: prometheus/templates/prometheus.yaml
	apiVersion: v1
	kind: Service
	metadata:
	labels:
	app: prometheus
	name: prometheus
	namespace: cilium-observability
	spec:
	ports:
	- name: webui
	port: 9090
	protocol: TCP
	targetPort: 9090
	selector:
	app: prometheus
	type: ClusterIP
	---
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	labels:
	app: grafana
	component: core
	name: grafana
	namespace: cilium-observability
	spec:
	replicas: 1
	selector:
	matchLabels:
	app: grafana
	template:
	metadata:
	labels:
	app: grafana
	spec:
	containers:
	- env:
	- name: GF_PATHS_CONFIG
	value: /configmap/grafana/grafana-config.ini
	- name: GF_PATHS_PROVISIONING
	value: /configmap/grafana/provisioning
	image: "docker.io/grafana/grafana:9.3.6"
	imagePullPolicy: IfNotPresent
	name: grafana-core
	readinessProbe:
	failureThreshold: 3
	httpGet:
	path: /login
	port: 3000
	scheme: HTTP
	volumeMounts:
	- mountPath: /configmap/grafana
	name: grafana-config
	readOnly: true
	dnsPolicy: ClusterFirst
	restartPolicy: Always
	volumes:
	- configMap:
	defaultMode: 420
	items:
	- key: grafana.ini
	path: grafana-config.ini
	- key: prometheus-datasource.yaml
	path: provisioning/datasources/prometheus.yaml
	- key: config.yaml
	path: provisioning/dashboards/config.yaml
	name: grafana-config
	name: grafana-config
	---
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	labels:
	app: prometheus
	name: prometheus
	namespace: cilium-observability
	spec:
	replicas: 1
	selector:
	matchLabels:
	app: prometheus
	template:
	metadata:
	labels:
	app: prometheus
	name: prometheus-main
	spec:
	containers:
	- args:
	- --config.file=/etc/prometheus/prometheus.yaml
	- --storage.tsdb.path=/prometheus/
	- --log.level=info
	- --enable-feature=exemplar-storage
	image: "prom/prometheus:v2.42.0"
	imagePullPolicy: IfNotPresent
	name: prometheus
	ports:
	- containerPort: 9090
	name: webui
	protocol: TCP
	volumeMounts:
	- mountPath: /etc/prometheus
	name: config-volume
	readOnly: true
	- mountPath: /prometheus/
	name: storage
	dnsPolicy: ClusterFirst
	restartPolicy: Always
	serviceAccount: prometheus-k8s
	volumes:
	- configMap:
	name: prometheus
	name: config-volume
	- emptyDir: {}
	name: storage
	hubble:
	enabled: true
	relay:
	enabled: true
	ui:
	enabled: true
	metrics:
	enabled:
	- "policy:sourceContext=app\|workload-name\|pod\|reserved-identity;destinationContext=app\|workload-name\|pod\|dns\|reserved-identity;labelsContext=source_namespace,destination_namespace"