1st functional Heartbeat server / client

heartbeat2.py

#!/usr/bin/python

# import sched, time
from time import time
from time import sleep
from random import randint
from threading import Timer

from urllib2 import urlopen

import sys # what?
import urllib # what? why?

from BaseHTTPServer import BaseHTTPRequestHandler,HTTPServer

# https://docs.python.org/2/library/sched.html
#### SH = sched.scheduler(time, sleep)

PORT_NUMBER = 8888
HOST_NAME = 'localhost'

# localhost:8888/hb_init?48&appid=test_client_python =>
#         /hb_init
#         [*] 
#         ['48', 'appid=test_client_python']
#         Accept-Encoding: identity
#         Host: localhost:8080
#         Connection: close
#         User-Agent: Python-urllib/2.7
#         
#         ID: Python-urllib/2.7@127.0.0.1

visits = {}
# overdue = 0 # Just for now... TODO: FIXME?: should be a part of visit data, right?!

def toolate(ID):
    ts = time() 
    print "[", ts, "] [", ID, "]: ", visits[ID]
    
    d = visits[ID]
    # Collect statistics about ID    
    
    if d[3] > 5:
        print("TOO overdue!") # TODO: early detection of overdue clients!!???
        del visits[ID]
    else:
        visits[ID] = (d[0], d[1], Timer(d[1], toolate, [ID]), d[3] + 1)
        visits[ID][2].start() # Another Chance???


class MyHandler(BaseHTTPRequestHandler):
#        s.headers, s.client_address, s.command, s.path, s.request_version, s.raw_requestline
    def status(s, ID):
        d = visits[ID]
        
        t =  d[0]
        dd = d[1]
        od = d[3]
        
        ## STATUS CODE???
        
        app = "application [" + str(ID) + "]| od="+str(od)+";1;3;0;10 delay="+str(dd)+";;; "
        
        if od == 0:
            s.wfile.write("OK - fine " + app)
        elif od > 0:
            s.wfile.write("WARNING - somewhat late " + app)
        elif od > 2:
            s.wfile.write("CRITICAL - somewhat late " + app)


    def do_GET(s):
        global visits
#        global overdue
        
        s.send_response(200)
        s.send_header('Content-type','text/html')
        s.send_header('Access-Control-Allow-Origin','*')
        s.end_headers()
        
        ### TODO: FIXME: the following url parsing is neither failsafe nor secure! :-(
        path, _, tail = s.path.partition('?')
        path = urllib.unquote(path)

        if path == "/list":
            for k, v in visits.iteritems():
                s.wfile.write(str(k) + "\n")
            return

        query = tail.split('&')
        
        if path == "/status":
            if tail != "":
                ID = query[0].split('=')[1] # + " @ " + s.client_address[0] # " || " + s.headers['User-Agent']
                if ID in visits:
                    s.status(ID)
                else:
                    s.wfile.write("CRITICAL - no application record for " + str(ID))
            else:
                if len(visits) == 1:
                    ID = visits.iterkeys().next()
                    s.status(ID)                    
                elif len(visits) > 1:
                    s.wfile.write("WARNING - multiple (" + str(len(visits)) + ") applications" )
                else:
                    s.wfile.write("UNKNOWN -  no heartbeat clients yet..." )
                    
            return
        
        
        # PARSING: s.path -->>> path ? T & appid = ID        
        T = int(query[0])
	ID = query[1].split('=')[1] + " @ " + s.client_address[0] # " || " + s.headers['User-Agent']
                
        if ID in visits:
            print "PREVIOUS STATE", visits[ID]
            visits[ID][2].cancel() # !
            
        ts = time() 

        
        if path == "/hb_init":
            # Hello little brother! Big Brother is watching you!
            print "Creation from scratch : ", ID, " at ", ts 
            T = T + 1 #max(10, (T*17)/16)
            visits[ID] = (ts, T, Timer(T, toolate, [ID]), 0)
            s.wfile.write(T) # ?
            visits[ID][2].start()            

        if path == "/hb_done":
            print "Destruction: ", ID, " at ", ts
            del visits[ID]
            s.wfile.write("So Long, and Thanks for All the Fish!")
	
        if path == "/hb_ping": #
            # TODO: make sure visits[ID] exists!
            print "HEART-BEAT for: ", ID, " at ", ts  # Here i come again... 
            lastts = visits[ID][0]
            lastt  = visits[ID][1]
            overdue = visits[ID][3]
#            if (ts - lastts) > lastt: # Sorry Sir, but you are too late :-(
#                overdue += 1
             
            if overdue > 3:
                print("overdue!") # TODO: early detection of overdue clients!!???
                s.wfile.write("dead") # ?
#                del visits[ID] #??
            else:
                T = T + 1 # max(3, (T*11)/8)
                visits[ID] = (ts, T, Timer(T, toolate, [ID]), overdue)
                s.wfile.write(T) # ?
                visits[ID][2].start()            
        # WHAT ELSE????    
        return 

def test_server(HandlerClass = MyHandler, ServerClass = HTTPServer, protocol="HTTP/1.0"):
    """Test the HTTP request handler class.
    """

    port = PORT_NUMBER
    server_address = (HOST_NAME, port)

    HandlerClass.protocol_version = protocol
    httpd = ServerClass(server_address, HandlerClass)

    sa = httpd.socket.getsockname()
    print "Serving HTTP on", sa[0], "port", sa[1], "..."
    httpd.serve_forever()


def test_client():
    t = randint(2, 5)
    APP_ID = "test_client_python%" + str(randint(99999999, 9999999999)) # TODO: get unique ID from server?
    HB_SERVER_URL = "http://" + HOST_NAME + ":" + str(PORT_NUMBER)

    print "List HB apps: " + urlopen(HB_SERVER_URL + "/list" ).read()
    print "APP HB Status: " + urlopen(HB_SERVER_URL + "/status" ).read()
    
    tt = urlopen(HB_SERVER_URL + "/hb_init?" + str(t) + "&appid="+ APP_ID ).read()
    print "Initial response: ", tt

#    print "List HB apps: " + urlopen(HB_SERVER_URL + "/list" ).read()
#    print "APP HB Status: " + urlopen(HB_SERVER_URL + "/status" ).read()
    
    overdue = 0
    
    i = 0
    for i in xrange(1, 25):
#    while True:    
        i = i + 1
        d = randint(0, (int(t) * 2)/1)
        
        try:
            print d, " > ", tt, "?"        
            if d > int(tt):
                overdue += 1
        except:
            pass        
            
        print "heart-beat: ", i, "! Promise: ", t, ", Max: ", tt, ", Delay: ", d, " sec........ overdues?: ", overdue
        sleep(d)
        
        # heartbeat: 
        t = randint(0, 5)
        
#        print "List HB apps: " + urlopen(HB_SERVER_URL + "/list" ).read()
#        print "APP HB Status: " + urlopen(HB_SERVER_URL + "/status" ).read()
        
        print "Ping: ", t
        tt = urlopen(HB_SERVER_URL + "/hb_ping?" + str(t) + "&appid="+ APP_ID ).read()
        print "Pong: ", tt
        
#        print "List HB apps: " + urlopen(HB_SERVER_URL + "/list" ).read()
#        print "APP HB Status: " + urlopen(HB_SERVER_URL + "/status" ).read()
        
        if tt == "dead": # Just for testing...
            
            print "Ups: we run out of time..."
            tt = urlopen(HB_SERVER_URL + "/hb_done?0"+ "&appid="+ APP_ID ).read()
            print "Goodbye message: ", tt

            print "List HB apps: " + urlopen(HB_SERVER_URL + "/list" ).read()
            print "APP HB Status: " + urlopen(HB_SERVER_URL + "/status" ).read()

            break
    
    
# port = int(sys.argv[1])
if __name__ == '__main__':
    print(sys.argv)
    if (len(sys.argv) == 1):
        test_client()
    else: # Any Arguments? => Start HB Server
#        if (sys.argv[1] == "-server"):
        test_server()

#####################################################################
### Initial HB design: https://gist.github.com/malex984/dbec16e9c7d88f295071
###

### Heartbeat:
###	webgl: add to render loop (initiate)
###	multithread / asynchron sending (should not block the application)
###     non-blocking i/o
###     how often: 1 beat per second (or less)?
###     TCPIP connection: leave open?
###
###     GET request
###     sending: add into URL "ID exhibit, heartbeat expectation" (selbstverpflichtung)
###     The server knows the IP (computer).
###
###     initial setup for monitoring software (also list of programs which run on which host)
###     startup via Nagios?
###
###     ----
###     maintenance mode could be added (you put station on maintenance), if a station is not expected to be running (in order to avoid automatic restart during maintenance)

### Heartbeat protocol:
### * pass protocol parameters into container via ENVIRONMENT VARIABLES, e.g.
###
###   - HEARTBEART_HTTP=http//IP:PORT/heartbeat.html?container=pong&next=%n
###     = application substitutes %n with minimal time until next heartbeart	(milliseconds) and sends GET request
###     = server answers with maximal time for next heartbeart (>minimal heartbeat time) (otherwise it will take some action)
###
###   - HEARTBEART_TCP=IP:PORT
###     = CLIENT: send minimal time until next heartbeat (ms)
###     = SERVER: send maximal time until next heartbeat (ms)
### 
### * ENVIRONMENT PARAMETERS are passed by url parameters into browser based applications some API exposed by electron for kiosk applications?
### 
### * when container is starting, the management system is waiting for some predefined time (15 seconds? same as regular waiting time when the app is running properly) before expecting the first heartbeat; afterwards the protocol is self tuning

### Heartbeat protocol implementation:
### 
### * asynchronous HTTP requests in JS
### * provide a JS library the ???

Currently HB Server port: 8888.
Moreover for simplicity we assume that HB Server runs on the client host and serves a single client host.
Therefore HB_URL = http://localhost:8888 for any client application.

All HB-Client communications are HTTP-GET-Request at HB_URL + "/" + HB_COMMAND + "?" + str(T) + "&appid=" + APP_ID.

• NOTE: T is the client's expected delay, APP_ID is a unique client application ID
• NOTE: The server responds with the maximum time it is going to wait
• NOTE: HB_COMMAND is hb_init or hb_ping or hb_done as follows:

1. Setup/Initialization: HB_COMMAND = hb_init. To be used by client-application-starter. NOTE: the initial delay T may be elevated in order to account for the client-application initial startup & initialization.
2. Regular Heartbeat ping/pong: HB_COMMAND = hb_ping. To be sent from within client-application' main loop.
3. Application closing: HB_COMMAND = hb_done. Last communication from application to HB Server. NOTE: delay has no special meaning here.

ps: the prototype above ( https://gist.github.com/malex984/dbec16e9c7d88f295071#file-heartbeat2-py ) contains:

• HB Server, which can be started with: python2 heartbeat2.py -s (any argument)
• HB Client, which can be started with: python2 heartbeat2.py (no arguments)
  (both Server and Client are to be run on a same host, i.e. currently localhost is used everywhere)

Please change

        s.send_response(200)
        s.send_header('Content-type','text/html')
        s.end_headers()

to

        s.send_response(200)
        s.send_header('Content-type','text/html')
        s.send_header('Access-Control-Allow-Origin','*')
        s.end_headers()

This is needed for sending GET requests in JavaScript since the web site and the heartbeat server use different urls (cross origin requests). See https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS

Thanks! I updated the above heartbeat2.py

I added HB support to application choosers: both bash shell and appchoo, see: malex984/appchoo@c0e1701

Here is my basic HTML/JS client:
https://gist.github.com/porst17/1b2076e6d1baf4375500

Heartbeats are triggered manually to be able to simulate delayed heartbeats etc.