Setup Prometheus

setup-prometheus.sh

#!/bin/bash

PROMETHEUS_VERSION=${PROMETHEUS_VERSION:-'2.27.1'}
ALERTMANAGER_VERSION=${ALERTMANAGER_VERSION:-'0.21.0'}
NODE_EXPORTER_VERSION=${NODE_EXPORTER_VERSION:-'1.1.2'}

if command -v yum ; then
    sudo yum update -y
    sudo yum install -y wget
fi

if command -v apt ; then
    sudo apt update
    sudo DEBIAN_FRONTEND=noninteractive apt dist-upgrade -y
    sudo apt install -y wget
fi

if id -u 'centos'; then
    USER='centos'
elif id -u 'fedora'; then
    USER='fedora'
elif id -u 'ubuntu'; then
    USER='ubuntu'
elif id -u 'debian'; then
    USER='debian'
else
    USER=$USER
fi


sudo mkdir -p /opt/prometheus /etc/prometheus /etc/prometheus/alerts /var/lib/prometheus/data
sudo mkdir -p /opt/alertmanager /etc/alertmanager /etc/alertmanager/templates
sudo mkdir -p /opt/node_exporter

sudo chown -R $USER:$USER /opt/prometheus /opt/alertmanager /opt/node_exporter

curl -L https://github.com/prometheus/alertmanager/releases/download/v$ALERTMANAGER_VERSION/alertmanager-$ALERTMANAGER_VERSION.linux-amd64.tar.gz -o /tmp/alertmanager-$ALERTMANAGER_VERSION.linux-amd64.tar.gz
tar xvzf /tmp/alertmanager-$ALERTMANAGER_VERSION.linux-amd64.tar.gz -C /opt/alertmanager --strip=1

#sudo docker run -p 9090:9090 -v /home/ubuntu/prometheus:/etc/prometheus prom/prometheus
curl -L https://github.com/prometheus/prometheus/releases/download/v$PROMETHEUS_VERSION/prometheus-$PROMETHEUS_VERSION.linux-amd64.tar.gz -o /tmp/prometheus-$PROMETHEUS_VERSION.linux-amd64.tar.gz
tar xvzf /tmp/prometheus-$PROMETHEUS_VERSION.linux-amd64.tar.gz -C /opt/prometheus --strip=1

curl -L https://github.com/prometheus/node_exporter/releases/download/v$NODE_EXPORTER_VERSION/node_exporter-$NODE_EXPORTER_VERSION.linux-amd64.tar.gz -o /tmp/node_exporter-$NODE_EXPORTER_VERSION.tar.gz
tar xvzf /tmp/node_exporter-$NODE_EXPORTER_VERSION.tar.gz -C /opt/node_exporter --strip=1

cat << EOF | sudo tee /etc/prometheus/prometheus.yml
global:
  scrape_interval:     15s
  evaluation_interval: 15s

# Alertmanager configuration
alerting:
  alertmanagers:
  - static_configs:
    - targets:
      - localhost:9093

rule_files:
  - "alerts/*.yml"

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090', 'localhost:9100']
  - job_name: 'test2'
    static_configs:
      - targets: ['test2-vm:9100']
EOF


cat << EOF | sudo tee /etc/alertmanager/alertmanager.yml
global:
  resolve_timeout: 1m
  smtp_from: 'prometheus@test.com'
  smtp_smarthost: 'poczta.test.com:587'
  smtp_auth_username: 'prometheus@test.com'
  smtp_auth_password: 'test'
  smtp_auth_secret: 'prometheus@test.com'

route:
  group_by: ['alertname', 'priority']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 1m
  receiver: 'some-email'

receivers:
  - name: "some-email"
    email_configs:
    - to: 'prometheus@test.com'
      send_resolved: true

inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']
EOF

cat << EOF | sudo tee /etc/prometheus/alerts/disk.yml
groups:
- name: diskUsage
  rules:
  - alert: diskusage
    expr: node_filesystem_free_bytes{mountpoint ="/"} / node_filesystem_size_bytes{mountpoint ="/"} * 100 > 50
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "Disk is out of space"
      description: "There is less space than 50 percent on rootfs!"
EOF

cat << EOF | sudo tee /etc/alertmanager/templates/warning.tmpl
warning!
EOF

cat << EOF | sudo tee /etc/systemd/system/prometheus.service
[Unit]
Description=Prometheus Server
Documentation=https://prometheus.io/docs/
After=network-online.target

[Service]
User=root
Restart=on-failure
EnvironmentFile=/etc/sysconfig/prometheus
ExecStart=/opt/prometheus/prometheus \$OPTIONS

[Install]
WantedBy=multi-user.target
EOF

cat << EOF | sudo tee /etc/systemd/system/alertmanager.service
[Unit]
Description=Alertmanager service for Prometheus
Documentation=https://prometheus.io/docs/
After=network-online.target

[Service]
User=root
Restart=on-failure
EnvironmentFile=/etc/sysconfig/alertmanager
ExecStart=/opt/alertmanager/alertmanager \$OPTIONS

[Install]
WantedBy=multi-user.target
EOF

cat << EOF | sudo tee /etc/systemd/system/node_exporter.service
[Unit]
Description=Prometheus Node Exporter
Wants=network-online.target
After=network-online.target

[Service]
User=root
Restart=on-failure
ExecStart=/opt/node_exporter/node_exporter \$OPTIONS
EnvironmentFile=/etc/sysconfig/node_exporter

[Install]
WantedBy=multi-user.target
EOF

cat << EOF | sudo tee /etc/sysconfig/prometheus
OPTIONS="--config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/data"
EOF

cat << EOF | sudo tee /etc/sysconfig/alertmanager
OPTIONS="--config.file=/etc/alertmanager/alertmanager.yml --storage.path=/var/lib/prometheus/alertmanager"
EOF

cat << EOF | sudo tee /etc/sysconfig/node_exporter
OPTIONS="--collector.systemd"
EOF

sudo systemctl daemon-reload
sudo systemctl enable prometheus
sudo systemctl enable alertmanager
sudo systemctl enable node_exporter
sudo systemctl start prometheus
sudo systemctl start alertmanager
sudo systemctl start node_exporter