EOS/Telos Find Missing Blocks (v2)

detective_missed.py

#!/usr/bin/python

#
# This script is a crude way to figure out who is missing what blocks from our block producer logs
# usage: egrep --line-buffered 'producer_plugin|start_block|produce_block' (/path/to/nodeos/log) | ./detective_missed.py
#
# Stop program on missing blocks found
stopOnFind = False

# Show tick on Good Block
showTick = True

# Show Schedule Proposals
showScheduleProposed = False

# Show Schedule Changes
showScheduleChange = True

# Truncate Data dumps
verbose = True

import sys
import iso8601
import re
import json

schedules = []
currentSchedule = None
lastResult = None

class Trail(object):
    def __init__(self):
        self.history = {}
    def add(self, block, result):
        if len(self.history.keys()) > 2000:
            self.history.pop(sorted(self.history.keys())[0])
        self.history[block] = result
    def get(self, block):
        if block in self.history : return(self.history[block])
        return(None)
    def check(self, low, high):
        if low > high:
            t = low
            low = high
            high = t
        for block in range(low, high, 1):
            if block not in self.history:
                return(False)
        return(True)

class Schedule(object):
    def __init__(self, line):
        self.line = line
        self.producers = []
        self.activeAt = int(re.sub("\)","",line[12]))
        schedule = json.loads(line[21])
        self.version = int(schedule['version'])
        for producer in schedule['producers']:
            self.producers.append(producer['producer_name'])

class Line(object):
    def __init__(self, line, schedule):
        self.time = int(iso8601.parse_date(line[11]).strftime('%s'))
        self.date = line[11]
        self.schedule = schedule
        self.block = int(re.sub("#", "", line[9]))
        self.producer = line[14]
        self.line = line


def on_incoming_block(last, line, schedule, trail):
    if len(line) < 11 : return(last)
    if line[11] == 'confirming' : return(last)
    result=Line(line, schedule)
    trail.add(result.block, result)
    if last is None : return(result)

    if abs(result.time - last.time)>6 and abs(result.block - last.block) != 1000:
        violators = []
        heroes = []
        if last.schedule is not None:
            fp = last.schedule.producers.index(last.producer)
            tp = last.schedule.producers.index(result.producer)
            producers = last.schedule.producers + last.schedule.producers
            if fp > tp:
                violators = producers[fp:tp+len(last.schedule.producers)+1]
            else:
                violators = producers[fp:tp+1]
            if len(violators)>1:
                heroes.append(violators.pop(0))
                heroes.append(violators.pop())
        else:
            violators = ['no schedule']
        print
        print "Time Error"
        print "delta: "+str(abs(result.time - last.time))+" block: "+str(last.block)+" heroes: "+json.dumps(heroes)+" violators: "+json.dumps(violators)

    blockDelta = abs(result.block - last.block)
    if blockDelta == 1000:
        sys.stdout.write('*')
        sys.stdout.flush()
        return(result)
    elif blockDelta > 1:
        if trail.check(last.block, result.block) : return(result)
        timeDelta = abs(result.time-last.time)
        heroes = []
        violators = []
        if last.schedule is None:
            violators = [ 'Error:', 'Schedule Unknown' ]
        elif last.schedule.producers.count(last.producer) == 0 or last.schedule.producers.count(result.producer) == 0:
            violators = [ 'Error:', 'producers not found on current schedule', 'last:', last.producer, 'result:', result.producer ]
        elif last.producer == result.producer:
            violators = [ last.producer ]
        else:
            fp = last.schedule.producers.index(last.producer)
            tp = last.schedule.producers.index(result.producer)
            producers = last.schedule.producers + last.schedule.producers
            if fp > tp:
                violators = producers[fp:tp+len(last.schedule.producers)+1]
            else:
                violators = producers[fp:tp+1]
            if len(violators)>1:
                heroes.append(violators.pop(0))
                heroes.append(violators.pop())
        print
        print 'Found Gap: delta: { block: '+str(blockDelta)+', time: '+str(timeDelta)+' }, heroes: '+json.dumps(heroes)+', violators: '+json.dumps(violators)+', from: { block: '+str(last.block)+', producer: '+last.producer+', date: '+last.date+' }, to: { block: '+str(result.block)+', producer: '+result.producer+', date: '+result.date+' }'
        l = 0
        h = 0
        if last.block > result.block:
            l = result.block
            h = last.block
        else:
            l = last.block
            h = result.block
        for b in range(l, h+1, 1):
            block = trail.get(b)
            if block is None:
                if verbose : print "#"+str(b)+": Missing"
            else:
                for p in range(1,8,1) : block.line.pop(0)
                if verbose : print "#"+str(b)+": "+json.dumps(block.line)
        if stopOnFind : raise Exception('Missing Blocks Found')
        return(result)
    else:
        if showTick : sys.stdout.write('.')
        sys.stdout.flush()
        return(result)

def start_block(line):
    schedule = Schedule(line)
    return(schedule)

trail = Trail()
while True:
    line=sys.stdin.readline().split()
    # Python needs a better switch/case
    if len(line) < 5 : continue
    if line[4] == "on_incoming_block" or line[4] == 'produce_block':
        if line[9] == 'is' or line[9] == 'block' : continue
        # activate schedule if at that block
        currentBlock = int(re.sub("#","",line[9]))
        for schedule in schedules:
            if schedule.activeAt < currentBlock:
                if showScheduleChange:
                    print
                    print "Schedule Changed"
                    if verbose : print str(schedule.version)+": "+json.dumps(schedule.producers)
                currentSchedule = schedule
                schedules.remove(schedule)
                break
        lastResult = on_incoming_block(lastResult, line, currentSchedule, trail)
        continue

    if line[4] == "start_block":
        s = start_block(line)
        for schedule in schedules:
            if s.version == schedule.version : s = None
            break
        if s is not None:
            schedules.append(s)
            if showScheduleProposed:
                print
                print "Schedule Proposed"
                if verbose : print str(s.version)+": "+json.dumps(s.producers)
        continue