metrics server

grafana.ini

[server]
# The ip address to bind to, empty will bind to all interfaces
http_addr = 127.0.0.1

# The http port  to use
http_port = 3000

# The public facing domain name used to access grafana from a browser
domain = {{ grafanaDomain }}

# Redirect to correct domain if host header does not match domain
# Prevents DNS rebinding attacks
enforce_domain = true

# The full public facing url you use in browser, used for redirects and emails
# If you use reverse proxy and sub path specify full url (with sub path)
root_url = https://{{ grafanaDomain }}

# Log web requests
;router_logging = false

# the path relative working path
;static_root_path = public

# enable gzip
enable_gzip = true

# https certs & key file
;cert_file =
;cert_key =

# Unix socket path
;socket =

[analytics]
# Server reporting, sends usage counters to stats.grafana.org every 24 hours.
# No ip addresses are being tracked, only simple counters to track
# running instances, dashboard and error counts. It is very helpful to us.
# Change this option to false to disable reporting.
reporting_enabled = false

# Set to false to disable all checks to https://grafana.net
# for new vesions (grafana itself and plugins), check is used
# in some UI views to notify that grafana or plugin update exists
# This option does not cause any auto updates, nor send any information
# only a GET request to http://grafana.com to get latest versions
;check_for_updates = true

# Google Analytics universal tracking code, only enabled if you specify an id here
;google_analytics_ua_id =

#################################### Security ####################################
[security]
# default admin user, created on startup
;admin_user = admin

# default admin password, can be changed before first start of grafana,  or in profile settings
;admin_password = admin

# used for signing
;secret_key = SW2YcwTIb9zpOOhoPsMm

# Auto-login remember days
;login_remember_days = 7
;cookie_username = grafana_user
;cookie_remember_name = grafana_remember

# disable gravatar profile images
;disable_gravatar = false

# data source proxy whitelist (ip_or_domain:port separated by spaces)
;data_source_proxy_whitelist =

# disable protection against brute force login attempts
;disable_brute_force_login_protection = false

#################################### Snapshots ###########################
[snapshots]
# snapshot sharing options
;external_enabled = true
;external_snapshot_url = https://snapshots-origin.raintank.io
;external_snapshot_name = Publish to snapshot.raintank.io

# remove expired snapshot
;snapshot_remove_expired = true

#################################### Dashboards History ##################
[dashboards]
# Number dashboard versions to keep (per dashboard). Default: 20, Minimum: 1
;versions_to_keep = 20

#################################### Users ###############################
[users]
# disable user signup / registration
;allow_sign_up = true

# Allow non admin users to create organizations
;allow_org_create = true

# Set to true to automatically assign new users to the default organization (id 1)
;auto_assign_org = true

# Default role new users will be automatically assigned (if disabled above is set to true)
;auto_assign_org_role = Viewer

# Background text for the user field on the login page
;login_hint = email or username

# Default UI theme ("dark" or "light")
;default_theme = dark

# External user management, these options affect the organization users view
;external_manage_link_url =
;external_manage_link_name =
;external_manage_info =

# Viewers can edit/inspect dashboard settings in the browser. But not save the dashboard.
;viewers_can_edit = false

[auth]
# Set to true to disable (hide) the login form, useful if you use OAuth, defaults to false
;disable_login_form = false

# Set to true to disable the signout link in the side menu. useful if you use auth.proxy, defaults to false
;disable_signout_menu = false

# URL to redirect the user to after sign out
;signout_redirect_url =

#################################### Anonymous Auth ##########################
[auth.anonymous]
# enable anonymous access
;enabled = false

# specify organization name that should be used for unauthenticated users
;org_name = Main Org.

# specify role for unauthenticated users
;org_role = Viewer

#################################### External image storage ##########################
[external_image_storage]
# Used for uploading images to public servers so they can be included in slack/email messages.
# you can choose between (s3, webdav, gcs, azure_blob, local)
;provider =

[external_image_storage.s3]
;bucket =
;region =
;path =
;access_key =
;secret_key =

[external_image_storage.webdav]
;url =
;public_url =
;username =
;password =

[external_image_storage.gcs]
;key_file =
;bucket =
;path =

[external_image_storage.azure_blob]
;account_name =
;account_key =
;container_name =

[external_image_storage.local]
# does not require any configuration

node_exporter.service

[Unit]
Description=node_exporter
After=network.target

[Service]
ExecStart=/srv/{{ prometheusNodeExporterPackage }}/node_exporter --web.listen-address="127.0.0.1:9100"
Type=simple
User=prometheus
Group=prometheus
Restart=always

[Install]
WantedBy=multi-user.target

playbook.hs

    , operation "Enable System Swap" $ \_ -> do
        enableSystemSwap 4096


    , operation "Update System Software" $ \_ -> do
        aptUpdate
        aptUpgrade


    , operation "Install prometheus node_exporter" $ \_ -> do
        unlessM (userExists "prometheus") $
            userAdd "prometheus"

        run_ "curl"
            [ "-L", "--progress-bar"
            , prometheusNodeExporterUrl
            , "-o", "/tmp/" <> prometheusNodeExporterPackage <> ".tar.gz"
            ]

        rm_rf $ "/srv/" <> fromText prometheusNodeExporterPackage
        run_ "tar" ["xf", "/tmp/" <> prometheusNodeExporterPackage <> ".tar.gz", "-C", "/srv"]
        run_ "rm" ["/tmp/" <> prometheusNodeExporterPackage <> ".tar.gz"]

        let env = fromPairs
                [ "prometheusNodeExporterPackage" .= prometheusNodeExporterPackage
                ]
        renderTemplateFile ("metrics-server/systemd/node_exporter.service") env "/etc/systemd/system/node_exporter.service"

        run_ "systemctl" ["daemon-reload"]
        run_ "systemctl" ["enable", "node_exporter.service"]
        run_ "systemctl" ["restart", "node_exporter.service"]


    , operation "Install prometheus" $ \_ -> do
        domains <- readVaultSecret vaultSecret_domains
        metricsBasicAuthUsers <- readVaultSecret vaultSecret_metricsBasicAuthUsers

        unlessM (userExists "prometheus") $
            userAdd "prometheus"

        mkdir_p "/var/lib/prometheus"
        run_ "chown" ["prometheus:prometheus", "/var/lib/prometheus"]

        let prometheusPackage = "prometheus-" <> prometheusVersion <> ".linux-amd64"
        run_ "curl"
            [ "-L", "--progress-bar"
            , "https://github.com/prometheus/prometheus/releases/download/v" <> prometheusVersion <> "/" <> prometheusPackage <> ".tar.gz"
            , "-o", "/tmp/" <> prometheusPackage <> ".tar.gz"
            ]

        rm_rf $ "/srv/" <> fromText prometheusPackage
        run_ "tar" ["xf", "/tmp/" <> prometheusPackage <> ".tar.gz", "-C", "/srv"]
        run_ "rm" ["/tmp/" <> prometheusPackage <> ".tar.gz"]

        let env1 = fromPairs
                [ "prometheusDomain" .= prometheusDomain domains
                , "wwwDomain" .= wwwDomain domains
                , "logsDomain" .= logsDomain domains
                , "apexDomain" .= apexDomain domains
                , "metricsUser" .= basicAuthUser (basicAuthFirstUser metricsBasicAuthUsers)
                , "metricsPassword" .= basicAuthPassword (basicAuthFirstUser  metricsBasicAuthUsers)
                ]
        renderTemplateFile ("metrics-server/prometheus.yml") env1 "/etc/prometheus.yml"

        let env2 = fromPairs
                [ "prometheusPackage" .= prometheusPackage
                , "prometheusDomain" .= prometheusDomain domains
                ]
        renderTemplateFile ("metrics-server/systemd/prometheus.service") env2 "/etc/systemd/system/prometheus.service"

        run_ "systemctl" ["daemon-reload"]
        run_ "systemctl" ["enable", "prometheus.service"]
        run_ "systemctl" ["restart", "prometheus.service"]


    , operation "Install grafana" $ \_ -> do
        domains <- readVaultSecret vaultSecret_domains

        -- Required dependency of grafana:
        aptInstall ["libfontconfig"]

        run_ "curl"
            [ "--progress-bar"
            , "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana_" <> grafanaVersion <> "_amd64.deb"
            , "-o", "/tmp/grafana.deb"
            ]
        run_ "dpkg" ["-i", "/tmp/grafana.deb"]

        let env = fromPairs
                [ "grafanaDomain" .= grafanaDomain domains
                ]
        renderTemplateFile ("metrics-server/grafana.ini") env "/etc/grafana/grafana.ini"

        run_ "systemctl" ["enable", "grafana-server.service"]
        run_ "systemctl" ["restart", "grafana-server.service"]

prometheus.service

[Unit]
Description=Prometheus
After=network.target

[Service]
ExecStart=/srv/{{ prometheusPackage }}/prometheus \
        --web.listen-address="127.0.0.1:9090" \
        --web.external-url="https://{{ prometheusDomain }}" \
	--config.file=/etc/prometheus.yml \
	--storage.tsdb.path=/var/lib/prometheus/data/ \
	--storage.tsdb.retention=30d
ExecReload=/bin/kill -HUP $MAINPID
Type=simple
User=prometheus
Group=prometheus
Restart=always

[Install]
WantedBy=multi-user.target

prometheus.yml

global:
  scrape_interval: 15s # Default is 1m

scrape_configs:
  - job_name: "prometheus"
    static_configs:
      - targets: ["localhost:9090"]

  - job_name: "prometheus_node"
    static_configs:
      - targets: ["localhost:9100"]
    relabel_configs:
      - target_label: "job"
        action: "replace"
        replacement: "node"
      - target_label: "instance"
        action: "replace"
        replacement: "{{ prometheusDomain }}:9100"

  - job_name: "www_node"
    scheme: "https"
    basic_auth:
      username: "{{ metricsUser }}"
      password: "{{ metricsPassword }}"
    static_configs:
      - targets: ["{{ wwwDomain }}:9100"]
    relabel_configs:
      - target_label: "job"
        action: "replace"
        replacement: "node"

  - job_name: "logs_node"
    scheme: "https"
    basic_auth:
      username: "{{ metricsUser }}"
      password: "{{ metricsPassword }}"
    static_configs:
      - targets: ["{{ logsDomain }}:9100"]
    relabel_configs:
      - target_label: "job"
        action: "replace"
        replacement: "node"

  - job_name: "game_node"
    scheme: "https"
    basic_auth:
      username: "{{ metricsUser }}"
      password: "{{ metricsPassword }}"
    dns_sd_configs:
      - names: ["_metrics._tcp.node.{{ apexDomain }}"]
    relabel_configs:
      - target_label: "job"
        action: "replace"
        replacement: "node"