Simple log-parsing pipeline

failfinder.py

#! /usr/bin/env python3

import argparse
import os


def check_folder(folder):
    logs = get_logs(folder)
    failures = collect_failures(logs)
    print_failures(failures)


def get_logs(folder):
    for entry in os.scandir(folder):
        if entry.is_dir():
            yield from get_logs(entry.path)
        elif entry.is_file() and not entry.is_symlink():
            yield entry.path


def collect_failures(paths):
    for filedescriptor in get_file_descriptors(paths):
        yield from get_failure_lines(filedescriptor)


def get_file_descriptors(paths):
    for path in paths:
        if not path.endswith('.log'):
            continue

        with open(path, 'r') as fd:
            yield fd


def get_failure_lines(filedescriptor):
    for linenumber, line in enumerate(filedescriptor):
        if 'failed' in line.lower():
            yield filedescriptor.name, linenumber, line


def print_failures(failures):
    for filename, linenumber, line in failures:
        print('{}, line {}: {}'.format(filename, linenumber, line.strip()))


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('folder')

    args = parser.parse_args()

    check_folder(args.folder)