script to parse openstack-ansible gate logs for task timings

gateparse.py

#!/usr/bin/env python

import argparse
import json
import re
import sys
import yaml

from datetime import datetime
from datetime import timedelta


class Task(object):
    def __init__(self, name, role, start, end):
        self.name = name
        self.role = role
        self.start = start
        self.end = end

    def elapsed(self):
        difference = self.end - self.start
        return difference


def group_play(lines):
    """Provides buffer of play output"""
    buffer = []
    for line in lines:
        if "PLAY [" in line:
            if buffer:
                yield buffer
            buffer = [line]
        else:
            buffer.append(line)
    yield buffer


def group_tasks(lines):
    """Provides buffer of task output"""
    buffer = []
    for line in lines:
        if "TASK:" in line:
            if buffer:
                yield buffer
            buffer = [line]
        elif len(buffer) > 0:
            buffer.append(line)
    yield buffer


def get_task(task_start, task_end):
    """Searches through lines of task output to create a task object"""
    # ridiculous regex to match line denoting a new task
    # named groups for each section within that line
    task_pattern = re.compile('^(?P<datetime>[\d\-]+ [\d\:\.]+)( \| TASK: \[(?:(?P<role>[\w\_\-]+) \| )?(?:(?P<task>[\w\-\_ \.\/\(\)\"\&\[\]\:\'\-\!\,\{\}]+))\])?')
    datetime_format = "%Y-%m-%d %H:%M:%S.%f"
    task_info = task_pattern.match(task_start)
    name = task_info.group('task')
    role = task_info.group('role')
    task_start_datetime = datetime.strptime(task_info.group('datetime'), datetime_format)
    start_datetime = task_info.group('datetime')
    task_end_info = task_pattern.match(task_end)
    task_end_datetime = datetime.strptime(task_end_info.group('datetime'), datetime_format)
    task = Task(name, role, task_start_datetime, task_end_datetime)
    return task


def dump_data(data, format):
    if format == 'yaml':
        return yaml.dump(data)
    elif format == 'json':
        return json.dumps(data)


def main(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument('file',
                        help='location to text file of gate log')
    parser.add_argument('--format', choices=['yaml', 'json'], default='yaml',
                        help='format in which to print parsed data')
    args = parser.parse_args()
    with open(args.file, 'r') as log:
        data = {}
        total_time = timedelta(0)
        play_number = 0
        for lines in group_play(log):
            play_time = timedelta(0)
            play = lines[0]
            tasks = lines[1:]
            task_number = 0
            play_pattern = '^[\d\-]+ [\d\:\.]+ \| PLAY \[(.*)\]'
            play_match = re.search(play_pattern, play)
            if play_match:
                play_name = play_match.group(1)
                data[play_number] = {}
                data[play_number]['name'] = play_name
                data[play_number]['tasks'] = []
                for task in group_tasks(tasks):
                    if len(task) > 0:
                        task = get_task(task[0], task[-1])
                        task_info = {}
                        task_info['id'] = task_number
                        task_info['name'] = task.name
                        task_info['role'] = task.role
                        task_info['time'] = str(task.elapsed())
                        data[play_number]['tasks'].append(task_info)
                        play_time = play_time + task.elapsed()
                        total_time = total_time + task.elapsed()
                        task_number = task_number + 1
                data[play_number]['time'] = str(play_time)
                play_number = play_number + 1
        data['total time'] = str(total_time)
        dumped_data = dump_data(data, args.format)
        print(dumped_data)

if __name__ == "__main__":
    main(sys.argv)