Heartbeats in the wrapper

heartbeat_logic.py

# 'p' is the Popen object for the subprocess.

end_time = time.time() + 2 * g_TO  # Time to initialize
all_is_well = True
missed_TO = 0

while True:
    try:
        (r, w, e) = zmq.select(
                [in_hb, in_cmd], [], [], timeout=end_time - time.time())

        for sock in r:
            msg = sock.recv() 

            if sock == in_hb:
                print '[monitor] Got heartbeat.'
                end_time = time.time() + g_TO
                missed_TO = 0

        if end_time <= time.time():
            end_time = time.time() + g_TO
            print '[monitor] Missing a timeout ...'
            missed_TO += 1

        if missed_TO >= g_N:
            print '[monitor] Too many TOs missed. Killing the process ...'
            retval = 'Missed ' + str(missed_TO) + ' heartbeats'
            all_is_well = False

    except zmq.ZMQError as e:
        print '[monitor] *** ZMQ had a problem: ' + str(e)

    if not all_is_well:
        if not _kill(p):
            print '[monitor] Probably already dead.'
        break