Finds longest resources in Chef-Solo's output logs (useful for finding long and potentially non idempotent ressources).

chef_log_optimizer.py

import re
import sys
import time

DATE_REGEX = re.compile(r'^\[(.*?)\].*$')
DATE_FORMAT = "%Y-%m-%dT%H:%M:%S+00:00"

# Find tasks longer than 3s
THRESHOLD = 3.0


def date_to_timestamp(datestr):
    return time.mktime(
        time.strptime(datestr, DATE_FORMAT)
    )


def extract_date(line):
    return DATE_REGEX.search(line).group(1)


def is_date(line):
    try:
        date_to_timestamp(extract_date(line))
        return True
    except:
        return False


def first(_list):
    return _list[0]


def last(_list):
    return _list[-1]


# Transform line, to tuple (timestamp, line)
def trans_line(line):
    try:
        return (
            date_to_timestamp(extract_date(line)),
            line
        )
    except Exception as e:
        print("line = %s" % repr(line))
        raise e


# Transforms pairs from
# (
#     (timestamp1, line1),
#     (timestamp2, line2)
# )
# to
#
# (timestamp2 - timestamp1, line1)


def trans_pairs(pair):
    t1, line1 = first(pair)
    t2, line2 = last(pair)
    return (t2 - t1, line1)


def beats_threshold(result):
    tdiff, line = result
    return tdiff > THRESHOLD


# Turns (1, 2, 3) into
# (
#   (1, 2),
#   (2, 3),
# )
def pairs(lst):
    i = iter(lst)
    first = prev = item = i.next()
    for item in i:
        yield prev, item
        prev = item
    #yield item, first



def main():
    # Read lines from stdin
    lines = [line for line in sys.stdin]

    # filter -> map -> reduce
    long_tasks = filter(beats_threshold,
        map(trans_pairs,
            pairs(
                map(trans_line,
                    filter(is_date, lines)
                )
            )
        )
    )

    # Print results
    for duration, task in sorted(long_tasks, key=first, reverse=True):
        print "%ss for %s" % (duration, task)

if __name__ == '__main__':
    main()