Add a posthook to prometheus pushgateway

gateway_hook.sh

# cat simple_posthook.sh
#/usr/bin/bash

PUSHGATEWAY_SERVER=_YOUR_PUSHGATEWAY_SERVER_ADDRESS_WITH_PORT
JOB_NAME=_YOUR_JOB_NAME
while true; do
  echo -e "HTTP/1.1 200 OK\n\n# $(date)" | nc -l 0.0.0.0 9092 &> /dev/null;
  sleep 3;
  curl -s "http://${PUSHGATEWAY_SERVER}/metrics" | sed -ne "s/push_time_seconds{instance=\"\([[:alnum:].]*\)\",job=\"${JOB_NAME}\".*/\1/p" | xargs -I INSNAME curl -X DELETE http://${PUSHGATEWAY_SERVER}/metrics/job/${JOB_NAME}/instance/INSNAME &> /dev/null;
done

optional_python_script_solution.py

import json
import time
import socket
import httplib
import SimpleHTTPServer
import SocketServer

PORT = 8000
HOSTNAME = socket.getfqdn()

class Handler(SimpleHTTPServer.SimpleHTTPRequestHandler):
    def do_GET(self):
        if self.path not in ("/", "/metrics"):
            return super(self, Handler).do_GET(self)

        conn = httplib.HTTPConnection("%s:9091" % HOSTNAME)
        conn.request('DELETE', '/metrics/job/log_collector', '')
        resp = conn.getresponse()
        print resp.status, resp.getheaders(), resp.read()
        time.sleep(1)
        self.protocol_version = 'HTTP/1.1'
        self.send_response(200, 'OK')
        self.send_header('Content-type', 'application/json')
        self.end_headers()
        self.wfile.write(bytes(json.dumps({"result": "ok"})))
        return


httpd = SocketServer.TCPServer(("", PORT), Handler)

print "serving at port", PORT
try:
    httpd.serve_forever()
except KeyboardInterrupt:
    pass
httpd.server_close()

prometheus.yml

- job_name: pushgateway
  honor_labels: true
  metrics_path: /metrics
  scheme: http
  static_configs:
  - targets:
    - PUSHGATEWAY_SERVER:9091
    - PUSHGATEWAY_HOOK_SERVER:9092

pushgateway-hook.service

[Unit]
Description=Pushgateway Post Hook
After=network-online.target firewalld.service
Wants=network-online.target

[Service]
ExecStart=/usr/bin/bash PATH_TO_gateway_hook.sh
TimeoutStartSec=0
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s

[Install]
WantedBy=multi-user.target