check_temporal_integrity-millis.py

#!/usr/bin/env python
import sys
import time
import signal
from optparse import OptionParser
from multiprocessing import Pool
from subprocess import Popen, PIPE

class TimeoutException(Exception):
  pass

def getMillis(host):
  def timeoutHandler(signum, frame):
    raise TimeoutException("getMillis for %s timed out after %d seconds" % (host, opts.timeout))

  millis=-1
  oldhandler = signal.signal(signal.SIGALRM, timeoutHandler)
  signal.alarm(opts.timeout)
  try:
    (seconds, nanos) = Popen (["/usr/bin/ssh", "-T", "-x", "-o","ControlMaster=auto","-o","ControlPath=/home/mklich/.ssh/deploy/88095cc0/%h:%p:%r", host, "date +%s.%N"], stdout=PIPE, stderr=PIPE).communicate()[0].strip('\n').split('.')
    millis = int(seconds) * 1000 + int(nanos) / 1000000
  except Exception as e:
    if opts.debug: print e
  finally:
    signal.signal(signal.SIGALRM, oldhandler)
    signal.alarm(0)
  if opts.debug:
    print host, "millis:", millis
  return (host, millis)

def updateMillis(args):
  for arg in args:
    if opts.debug: print "updating millis: %s (%d)" % (arg[0], arg[1])
    millis[arg[0]] = arg[1]

def getMinMax(dates):
  minhost = maxhost = ""
  for host, millis in dates.iteritems():
    if minhost == "":
      minhost = host
      maxhost = host
    else:
      if millis < dates[minhost]:
        minhost = host
      elif millis > dates[maxhost]:
        maxhost = host
  if opts.debug: print "min %s, max %s" % (minhost, maxhost)
  return (minhost, maxhost)

if __name__ == "__main__":
  usage = "usage: %prog [options] <host> [<host>...]"
  parser = OptionParser(usage=usage)
  parser.add_option("-s", "--millis", type="int", dest="threshold", default=5000, help="temporal disparity allowed (in millis)")
  parser.add_option("-t", "--timeout", type="int", dest="timeout", default=5, help="timeout for getting temporal value from host")
  parser.add_option("-d", "--debug", dest="debug", action="store_true", default=False, help="enable debug output")
  parser.add_option("-c", "--count", type="int", dest="check_count", default=1, help="how many times to run the check")
  (opts, hosts) = parser.parse_args()
  if opts.debug: print "opts: ", opts
  if opts.debug: print "hosts:", hosts

  for _ in range(opts.check_count):
    millis = {}
    message = ""
    exitcode = 0

    p = Pool(len(hosts))
    result = p.map_async(getMillis, hosts, chunksize=1, callback=updateMillis)
    while not result.ready():
        time.sleep(.5)
    (minhost, maxhost) = getMinMax(millis)
    timediff = millis[maxhost] - millis[minhost]
    if timediff < opts.threshold:
        message = "OK: time diff %d ms" % timediff
    else:
        message = "CRIT: time diff %ds - min: %s (%s)  max: %s (%s)" % (timediff,
            minhost, -1 if millis[minhost] == -1 else time.strftime("%H:%M:%S", time.localtime(millis[minhost]/1000)),
            maxhost, -1 if millis[maxhost] == -1 else time.strftime("%H:%M:%S", time.localtime(millis[maxhost]/1000)))
        exitcode = 2
    print message
  sys.exit(exitcode)