prometheus/prometheus#1610 reproduction script

prometheus-1610.sh

#!/bin/sh

set -eufx

tmpdir=$(mktemp -d ./prometheus-1610.XXXX)

cd "${tmpdir}"

cleanup() {
  set +e
  docker-compose down
  rm -rf "${tmpdir}"
}
trap cleanup EXIT

cat > docker-compose.yml <<'EOF'
version: "2"

services:
  app:
    image: nginx
    ports:
    - 80
    networks:
    - ws
  app2:
    image: nginx
    ports:
    - 80
    networks:
    - ws
  prometheus:
    image: prom/prometheus:v1.0.1
    command:
    - "-config.file=/etc/prometheus/prometheus.yml"
    - "-storage.local.path=/prometheus"
    ports:
    - '9090:9090'
    volumes:
    - './config:/etc/prometheus'
    - './data/prometheus:/prometheus'
    networks:
    - ws

networks:
  ws:
    driver: bridge
EOF

mkdir -p data/prometheus
mkdir -p config
cat > config/prometheus.yml <<'EOF'
global:
  scrape_interval:     5s
  evaluation_interval: 5s
scrape_configs:
  - job_name: 'myjob'
    dns_sd_configs:
    - names:
      - app
      type: A
      port: 80
      refresh_interval: 1s
EOF

docker-compose up -d
docker-compose scale app=2 app2=3

sleep 10

sed -i -e 's/app/app2/' config/prometheus.yml

docker-compose kill -s HUP prometheus

trap - EXIT

docker-compose ps

cat <<EOF
Prometheus is now on http://localhost:9090/

To shut down:

( cd "${tmpdir}" && docker-compose down ) && rm -rf "${tmpdir}"
EOF