scribe_monitor.py

#!/usr/bin/env python
'''
Scribe checker with heartbeat

desc: check scribe alive status, alert if fail
'''

import os
import sys
import time
import traceback
from scribe import scribe
from thrift.transport import TTransport, TSocket
from thrift.protocol import TBinaryProtocol

# config
HOST = '0.0.0.0'
PORT = 1456
CATEGORY = 'heartbeat_sg'
LOG_BASEDIR = '/var/log/scribe/default'
DELAY_WAIT = 10


class ScribeLogger(object):
    def __init__(self, host, port):
        self.host = host
        self.port = port
        self.succeed = False

    def log(self, category, message):
        log_entry = scribe.LogEntry(category=category, message=message)

        socket = TSocket.TSocket(host=self.host, port=self.port)
        transport = TTransport.TFramedTransport(socket)
        protocol = TBinaryProtocol.TBinaryProtocol(trans=transport, strictRead=False, strictWrite=False)
        client = scribe.Client(iprot=protocol, oprot=protocol)

        transport.open()
        result = client.Log(messages=[log_entry])
        transport.close()
        
        self.succeed = (result == scribe.ResultCode.OK)


class ScribeMonitor(object):
    def __init__(self, configdict):
        self.delay_wait = configdict.get('delay_wait')
        self.category = configdict.get('category')
        self.basedir = configdict.get('basedir')

        self.host = configdict.get('host')
        self.port = configdict.get('port')
        self.logger = ScribeLogger(self.host, self.port)

        self.entry_success = configdict.get('entry_success')
        self.entry_fail = configdict.get('entry_fail')
        
        self.status = 'init'

    @classmethod
    def get_ts_string(cls):
        return str(time.time())

    def _check(self):
        ts = self.get_ts_string()
        try:
            self.logger.log(self.category, ts)
        except:
            self.status = 'call_log_exception'
            print traceback.print_exc()
            return
        if self.logger.succeed == False:
            self.status = 'call_log_return_error'
            return
        time.sleep(self.delay_wait)
        log_file = self.basedir + '/%s/%s_current' % (self.category, self.category)
        tail_result = os.popen('tail -n1 %s' % log_file).read().strip()
        if tail_result != ts:
            # avoid night rotate reason, check yesterday's log
            import datetime
            yesterday = datetime.date.today() - datetime.timedelta(1)
            # TODO: find last log file, sequence not 00000
            log_file = self.basedir + '/%s/%s-%s_00000' % (self.category, self.category, yesterday.strftime('%Y-%m-%d')) 
            tail_result = os.popen('tail -n1 %s' % log_file).read().strip()
            if tail_result != ts:
                # report error
                self.status = 'log_miss'
                return
        self.status = 'normal'

    def _notice(self):
        if self.status == 'normal':
            self.entry_success(self.status)
            return True
        self.entry_fail(self.status)
        return False


    def run(self):
        self._check()
        return self._notice()


def success(status):
    pass

def fail(status):
    print status

if  __name__ == '__main__':
    configdict = dict(delay_wait=DELAY_WAIT, category=CATEGORY, host=HOST, port=PORT, basedir=LOG_BASEDIR, entry_success=success, entry_fail=fail)
    sm = ScribeMonitor(configdict)
    if sm.run():
        sys.exit(1)