Installing Prometheus on Debian

README.md

Install Prometheus on Debian

install gosu

just to not wrestle with sudo and exec

$ sudo apt-get update
$ sudo apt-get install gosu

Create prometheus user

$ sudo useradd -M prometheus
$ sudo usermod -L prometheus

Make scaffolding

$ sudo sudo mkdir /etc/prometheus
$ sudo sudo mkdir /var/lib/prometheus
$ sudo chown prometheus:prometheus /etc/prometheus
$ sudo chown prometheus:prometheus /var/lib/prometheus

Download prometheus

$ cd ~
$ wget https://github.com/prometheus/prometheus/releases/download/v2.0.0/prometheus-2.0.0.linux-amd64.tar.gz
$ sha256sum prometheus-2.0.0.linux-amd64.tar.gz 
e12917b25b32980daee0e9cf879d9ec197e2893924bd1574604eb0f550034d46  prometheus-2.0.0.linux-amd64.tar.gz
$ tar xvf prometheus-2.0.0.linux-amd64.tar.gz

Put bits of prometheus in places

$ sudo cp prometheus-2.0.0.linux-amd64/prometheus /usr/local/bin/
$ sudo cp prometheus-2.0.0.linux-amd64/promtool /usr/local/bin/
$ sudo chown prometheus:prometheus /usr/local/bin/prometheus
$ sudo chown prometheus:prometheus /usr/local/bin/promtool

$ sudo cp -r prometheus-2.0.0.linux-amd64/prometheus.yml /etc/prometheus/
$ sudo cp -r prometheus-2.0.0.linux-amd64/consoles /etc/prometheus
$ sudo cp -r prometheus-2.0.0.linux-amd64/console_libraries /etc/prometheus
$ sudo chown -R prometheus:prometheus /etc/prometheus/prometheus.yml
$ sudo chown -R prometheus:prometheus /etc/prometheus/consoles
$ sudo chown -R prometheus:prometheus /etc/prometheus/console_libraries

Configure prometheus

$ sudo vi /etc/prometheus/prometheus.yml

global:
  scrape_interval: 15s

scrape_configs:
  - job_name: 'prometheus'
    scrape_interval: 5s
    static_configs:
      - targets: ['localhost:9090']

Test if prometheus runs

$ sudo -u prometheus /usr/local/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/data --web.console.templates=/etc/prometheus/consoles --web.console.libraries=/etc/prometheus/consoles_libraries 
...
level=info ts=2018-01-17T17:34:29.810960616Z caller=main.go:371 msg="Server is ready to receive requests."

Ctrl+C

Add it as sysvinit service

See prometheus.init.d below

$ sudo cp prometheus.init.d /etc/init.d/prometheus
$ sudo chmod +x /etc/init.d/prometheus
$ sudo update-rc.d prometheus defaults
$ sudo service prometheus status
● prometheus.service - LSB: monitoring system and time series database.
   Loaded: loaded (/etc/init.d/prometheus)
   Active: inactive (dead)
$ sudo vi /etc/default/prometheus
START=yes

Start and check

$ sudo service prometheus start
$ ps auxww | grep prometheus
$ cat /var/log/prometheus/prometheus.log
$ sudo service prometheus stop

Download and install node_exporter

$ wget https://github.com/prometheus/node_exporter/releases/download/v0.15.2/node_exporter-0.15.2.linux-amd64.tar.gz
$ sha256sum node_exporter-0.15.2.linux-amd64.tar.gz 
1ce667467e442d1f7fbfa7de29a8ffc3a7a0c84d24d7c695cc88b29e0752df37  node_exporter-0.15.2.linux-amd64.tar.gz
$ tar xvf node_exporter-0.15.1.linux-amd64.tar.gz

$ sudo cp node_exporter-0.15.2.linux-amd64/node_exporter /usr/local/bin/
$ sudo chown prometheus:prometheus /usr/local/bin/node_exporter

Test if node_exporter runs

$ sudo -u prometheus /usr/local/bin/node_exporter --collector.loadavg --collector.meminfo --collector.filesystem
..
INFO[0000] Listening on :9100                            source="node_exporter.go:76"

Add node_exporter as sysvinit service

See promethnode_exportereus.init.d below

$ sudo cp node_exporter.init.d /etc/init.d/node_exporter
$ sudo chmod +x /etc/init.d/node_exporter
$ sudo update-rc.d node_exporter defaults
$ sudo service node_exporter status

$ sudo vi /etc/default/node_exporter
START=yes

Start and check

$ sudo service node_exporter start
$ ps auxww | grep node_exporter
$ cat /var/log/prometheus/node_exporter.log

Make prometheus to scrape node exporter

$ sudo vi /etc/prometheus/prometheus.yml

global:
  scrape_interval: 15s

scrape_configs:
  - job_name: 'prometheus'
    scrape_interval: 5s
    static_configs:
      - targets: ['localhost:9090']
  - job_name: 'node_exporter'
    scrape_interval: 5s
    static_configs:
      - targets: ['localhost:9100']

$ sudo service prometheus start

Further reading:

http://docs.grafana.org/installation/debian/

node_exporter.init.d

#!/bin/sh

### BEGIN INIT INFO
# Provides:          node_exporter
# Required-Start:    $remote_fs
# Required-Stop:     $remote_fs
# Should-Start:      $all
# Should-Stop:       $all
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Exporter for machine metrics.
# Description:       Prometheus exporter for machine metrics,
#                    written in Go with pluggable metric collectors.
### END INIT INFO

set -e

. /lib/lsb/init-functions

NAME=node_exporter
DESC="Exporter for machine metrics"
DAEMON=/usr/local/bin/node_exporter
USER=prometheus
CONFIG=
PID="/var/run/prometheus/$NAME.pid"
LOG="/var/log/prometheus/$NAME.log"
GOSU=/usr/sbin/gosu

ALERTMANAGER_OPTS=

DAEMON_OPTS="$ALERTMANAGER_OPTS"
DAEMON_OPTS="$DAEMON_OPTS --collector.loadavg --collector.meminfo --collector.filesystem --collector.diskstats.ignored-devices=\"^(ram|loop|fd)\\d+$\""

# Check if DAEMON binary exist
[ -f $DAEMON ] || exit 0

[ -f "/etc/default/$NAME" ] && . /etc/default/$NAME


service_not_configured () {
  if [ "$1" != "stop" ]; then
    printf "\tPlease configure $NAME and then edit /etc/default/$NAME\n"
    printf "\tand set the \"START\" variable to \"yes\" in order to allow\n"
    printf "\t$NAME to start.\n"
  fi
  exit 0
}

service_checks () {
  # Check if START variable is set to "yes", if not we exit.
  if [ "$START" != "yes" ]; then
    service_not_configured $1
  fi

  # Prepare directories
  mkdir -p "/var/run/prometheus" "/var/log/prometheus"
  chown -R $USER "/var/run/prometheus" "/var/log/prometheus"

  # Check if PID exists
  if [ -f "$PID" ]; then
    PID_NUMBER=`cat $PID`
    if [ -z "`ps axf | grep ${PID_NUMBER} | grep -v grep`" ]; then
      echo "Service was aborted abnormally; clean the PID file and continue..."
      rm -f "$PID"
    else
      echo "Service already started; skip..."
      exit 1
    fi
  fi
}

start () {
  service_checks $1
  $GOSU $USER $DAEMON $DAEMON_OPTS  > $LOG 2>&1  &
  RETVAL=$?
  echo $! > $PID

  if [ $RETVAL ]; then
    log_end_msg 0
  else
    log_end_msg 1
  fi
}

stop () {
  if start-stop-daemon --retry TERM/5/KILL/5 --oknodo --stop --quiet --pidfile $PID 2>&1 1>$LOG
  then
    log_end_msg 0
    rm $PID
  else
    log_end_msg 1
  fi
}


case "$1" in
  start)
    log_daemon_msg "Starting $DESC -" "$NAME"
    start
    ;;

  stop)
    log_daemon_msg "Stopping $DESC -" "$NAME"
    stop
    ;;

  reload)
    log_daemon_msg "Reloading $DESC configuration -" "$NAME"
    #-- sorry but node_exporter doesn't handle -HUP signal...
    #if start-stop-daemon --stop --signal HUP --quiet --oknodo --pidfile $PID --startas /bin/bash -- -c "exec $DAEMON $DAEMON_OPTS > $LOG 2>&1"
    #then
    #  log_end_msg 0
    #else
    #  log_end_msg 1
    #fi
    stop
    start
    ;;

  restart|force-reload)
    log_daemon_msg "Restarting $DESC -" "$NAME"
    stop
    start
    ;;

  syntax)
    $DAEMON --help
    ;;

  status)
    status_of_proc -p $PID $DAEMON $NAME
    ;;

  *)
    log_action_msg "Usage: /etc/init.d/$NAME {start|stop|reload|restart|force-reload|syntax|status}"
    ;;
esac

exit 0

prometheus.init.d

#!/bin/sh

### BEGIN INIT INFO
# Provides:          prometheus
# Required-Start:    $remote_fs
# Required-Stop:     $remote_fs
# Should-Start:      $all
# Should-Stop:       $all
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: monitoring system and time series database.
# Description:       Prometheus is a systems and service monitoring system.
#                    It collects metrics from configured targets at given intervals,
#                    evaluates rule expressions, displays the results,
#                    and can trigger alerts if some condition is observed to be true.
### END INIT INFO

set -e

. /lib/lsb/init-functions


NAME=prometheus
DESC="Prometheus monitoring system"
DAEMON=/usr/local/bin/prometheus
USER=prometheus
CONFIGDIR=/etc/prometheus
DATADIR=/var/lib/prometheus/data
PID="/var/run/prometheus/$NAME.pid"
LOG="/var/log/prometheus/$NAME.log"
GOSU=/usr/sbin/gosu

ALERTMANAGER_OPTS=

DAEMON_OPTS="$ALERTMANAGER_OPTS"
DAEMON_OPTS="$DAEMON_OPTS --config.file=$CONFIGDIR/prometheus.yml --storage.tsdb.path=$DATADIR"
DAEMON_OPTS="$DAEMON_OPTS --web.console.templates=$CONFIGDIR/consoles --web.console.libraries=$CONFIGDIR/console_libraries"

# Check if DAEMON binary exist
[ -f $DAEMON ] || exit 0

[ -f "/etc/default/$NAME" ] && . /etc/default/$NAME


service_not_configured () {
  if [ "$1" != "stop" ]; then
    printf "\tPlease configure $NAME and then edit /etc/default/$NAME\n"
    printf "\tand set the \"START\" variable to \"yes\" in order to allow\n"
    printf "\t$NAME to start.\n"
  fi
  exit 0
}

service_checks () {
  # Check if START variable is set to "yes", if not we exit.
  if [ "$START" != "yes" ]; then
    service_not_configured $1
  fi

  # Prepare directories
  mkdir -p "/var/run/prometheus" "/var/log/prometheus"
  chown -R $USER "/var/run/prometheus" "/var/log/prometheus"

  # Check if PID exists
  if [ -f "$PID" ]; then
    PID_NUMBER=`cat $PID`
    if [ -z "`ps axf | grep ${PID_NUMBER} | grep -v grep`" ]; then
      echo "Service was aborted abnormally; clean the PID file and continue..."
      rm -f "$PID"
    else
      echo "Service already started; skip..."
      exit 1
    fi
  fi
}

start () {
  service_checks $1
  $GOSU $USER   $DAEMON $DAEMON_OPTS  > $LOG 2>&1  &
  RETVAL=$?
  echo $! > $PID

  if [ $RETVAL ]; then
    log_end_msg 0
  else
    log_end_msg 1
  fi
}

stop () {
  if start-stop-daemon --retry TERM/5/KILL/5 --oknodo --stop --quiet --pidfile $PID 2>&1 1>$LOG
  then
    log_end_msg 0
    rm $PID
  else
    log_end_msg 1
  fi
}


case "$1" in
  start)
    log_daemon_msg "Starting $DESC -" "$NAME"
    start
    ;;

  stop)
    log_daemon_msg "Stopping $DESC -" "$NAME"
    stop
    ;;

  reload)
    log_daemon_msg "Reloading $DESC configuration -" "$NAME"
    if start-stop-daemon --stop --signal HUP --quiet --oknodo --pidfile $PID --startas /bin/bash -- -c "exec $DAEMON $DAEMON_OPTS > $LOG 2>&1"
    then
      log_end_msg 0
    else
      log_end_msg 1
      fi
    ;;

  restart|force-reload)
    log_daemon_msg "Restarting $DESC -" "$NAME"
    stop
    start
    ;;

  syntax)
    $DAEMON --help
    ;;

  status)
    status_of_proc -p $PID $DAEMON $NAME
    ;;

  *)
    log_action_msg "Usage: /etc/init.d/$NAME {start|stop|reload|restart|force-reload|syntax|status}"
    ;;
esac

exit 0

the second "See promethnode_exportereus.init.d below" should be "See node_exporter.init.d below"