icinga command wrapper

soft_recovery.py

#!/usr/bin/env python
# -* coding: utf-8 -*-

import sys
import subprocess
import optparse
import sqlite3

parser=optparse.OptionParser(usage='usage: %prog [options] check_cmd')
parser.disable_interspersed_args()
parser.add_option("--hostname", help='Short name for the host (i.e. "biglinuxbox").')
parser.add_option("--lasthoststate", help='A string indicating the last state of the host ("UP", "DOWN", or "UNREACHABLE").')
parser.add_option("--hostattempt", type="int", help='The number of the current host check retry.')
parser.add_option("--maxhostattempts", type="int", help='The max check attempts as defined for the current host.')
opts,arguments=parser.parse_args()

hostname = opts.hostname
lasthoststate = opts.lasthoststate
hostattempt = opts.hostattempt
maxhostattempts = opts.maxhostattempts

if not hostname:
    parser.error('--hostname not given')
elif not lasthoststate:
    parser.error('--lasthoststate not given')
elif not hostattempt:
    parser.error('--hostattempt not given')
elif not maxhostattempts:
    parser.error('--maxhostattempts not given')
elif not arguments:
    parser.error('check_cmd not given')

db_name = '/var/tmp/host_state.db'
table_name = 'soft_attempt'

conn = sqlite3.connect(db_name)
c = conn.cursor()
sql = """
        CREATE TABLE IF NOT EXISTS %s(
        hostname VARCHAR(255) PRIMARY KEY,
        attempt INTEGER NOT NULL)
        """ % table_name
c.execute(sql)

def select_attempt(hostname):
    c.execute("SELECT attempt FROM " + table_name + " WHERE hostname=?", (hostname,))
    row = c.fetchone()
    if row:
        attempt = row[0]
    else:
        attempt = None
    return attempt

def update_attempt(hostname, attempt):
    c.execute("UPDATE " + table_name + " SET attempt = ? WHERE hostname=?", (attempt, hostname))
    conn.commit()

def insert_attempt(hostname, attempt):
    c.execute("INSERT OR IGNORE INTO " + table_name + " (hostname, attempt) VALUES(?, ?)", (hostname, attempt))
    c.execute("UPDATE " + table_name + " SET attempt = ? WHERE hostname=?", (attempt, hostname))
    conn.commit()

def execute_check(args):
    p = subprocess.Popen(args, shell=True,
                        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout, stderr = p.communicate()
    returncode = p.returncode
    if len(stderr) > 0:
        return stderr.strip(), returncode
    else:
        return stdout.strip(), returncode

def hard_state_ok_attempt(hostname, hostattempt, maxhostattempts):
    # SOFT
    attempt = None
    nextattempt = maxhostattempts - hostattempt
    if lasthoststate == "DOWN" and nextattempt == 0 and returncode !=0:
        attempt = 0
        insert_attempt(hostname, attempt)
    else:
        attempt = select_attempt(hostname)
    return attempt

# Main
hostoutput, returncode = execute_check(" ".join(arguments))
hostoutput = hostoutput.split("|")

attempt = hard_state_ok_attempt(hostname, hostattempt, maxhostattempts)

# HARD state
if attempt is not None:
    # SOFT recovery from a HARD state
    if returncode == 0:
        # increment, update
        attempt += 1
        update_attempt(hostname, attempt)

        # Add OK info
        ok_info = "SOFT [%s/%s] -" % (attempt, maxhostattempts)
        hostoutput[0] = hostoutput[0].replace("-", ok_info, 1)
        print "|".join(hostoutput)
        # Set UP state
        if attempt >= maxhostattempts:
            c.execute("DELETE FROM " + table_name + " WHERE hostname=?", (hostname, ))
            conn.commit()
            sys.exit(returncode)
        # Set DOWN state
        else:
            sys.exit(2)
    elif returncode < 3:
        # Fail - reset attempt
        attempt = 0
        update_attempt(hostname, attempt)
        print "|".join(hostoutput)
        sys.exit(2)
    else:
        print "|".join(hostoutput)
        sys.exit(3)
# SOFT state
else:
    print "|".join(hostoutput)
    sys.exit(returncode)