flakes.py

from csv import writer
from os import remove
from pprint import pprint
from tempfile import mkstemp
from urllib.request import urlopen, URLError


urls = [
    'https://storage.googleapis.com/origin-ci-test/pr-logs/pull/21863/pull-ci-openshift-origin-master-e2e-aws/2968/build-log.txt',
    'https://storage.googleapis.com/origin-ci-test/pr-logs/pull/21739/pull-ci-openshift-origin-master-e2e-aws/2969/build-log.txt',
    'https://storage.googleapis.com/origin-ci-test/pr-logs/pull/21869/pull-ci-openshift-origin-master-e2e-aws/2970/build-log.txt',
    'https://storage.googleapis.com/origin-ci-test/pr-logs/pull/21863/pull-ci-openshift-origin-master-e2e-aws/2971/build-log.txt',
    'https://storage.googleapis.com/origin-ci-test/pr-logs/pull/21851/pull-ci-openshift-origin-master-e2e-aws/2973/build-log.txt',
    'https://storage.googleapis.com/origin-ci-test/pr-logs/pull/21863/pull-ci-openshift-origin-master-e2e-aws/2974/build-log.txt',
    'https://storage.googleapis.com/origin-ci-test/pr-logs/pull/21739/pull-ci-openshift-origin-master-e2e-aws/2975/build-log.txt',
    'https://storage.googleapis.com/origin-ci-test/pr-logs/pull/21871/pull-ci-openshift-origin-master-e2e-aws/2976/build-log.txt',
    'https://storage.googleapis.com/origin-ci-test/pr-logs/pull/21863/pull-ci-openshift-origin-master-e2e-aws/2979/build-log.txt',
    'https://storage.googleapis.com/origin-ci-test/pr-logs/pull/21739/pull-ci-openshift-origin-master-e2e-aws/2981/build-log.txt',
    'https://storage.googleapis.com/origin-ci-test/pr-logs/pull/21863/pull-ci-openshift-origin-master-e2e-aws/2982/build-log.txt',
    'https://storage.googleapis.com/origin-ci-test/pr-logs/pull/21868/pull-ci-openshift-origin-master-e2e-aws/2984/build-log.txt',
    'https://storage.googleapis.com/origin-ci-test/pr-logs/pull/21739/pull-ci-openshift-origin-master-e2e-aws/2986/build-log.txt',
    'https://storage.googleapis.com/origin-ci-test/pr-logs/pull/21869/pull-ci-openshift-origin-master-e2e-aws/2987/build-log.txt',
    'https://storage.googleapis.com/origin-ci-test/pr-logs/pull/21739/pull-ci-openshift-origin-master-e2e-aws/2988/build-log.txt',
]

def download(url):
    try:
        print("Downloading {}...".format(url))
        response = urlopen(url, timeout = 5)
        content = response.read()
        _, name = mkstemp()
        with open(name, 'wb') as file:
            file.write(content)
        return name
    except URLError:
        return ""

def list_problems(filename):
    flake_flag = False
    failure_flag = False
    flakes = set()
    failures = set()
    print("Analyzing {}...".format(filename))
    with open(filename) as file:
        for line in file:
            line = line.strip()
            if line == "":
                continue
            if line.startswith("Failing tests:"):
                failure_flag = True
                continue
            if line.startswith("Flaky tests:"):
                flake_flag = True
                continue
            if (flake_flag or failure_flag) and line.startswith("["):
                if flake_flag:
                    flakes.add(line)
                if failure_flag:
                    failures.add(line)
            else:
                flake_flag = False
                failure_flag = False

    return flakes, failures

def analyze_data(data):
    flakes = {}
    failures = {}
    urls = []
    iteration = 1
    for key in data:
        value = data[key]
        urls.append(key)
        flakes = analyze_lines(flakes, value[0], iteration)
        failures = analyze_lines(failures, value[1], iteration)
        iteration = iteration + 1
    return urls, flakes, failures

def analyze_lines(old, new, iteration):
    for line in new:
        if line not in old:
            old[line] = ["" for _ in range(iteration-1)]
        old[line].append("x")
    for key in old:
        value = old[key]
        if len(value) < iteration:
            value.extend(["" for _ in range(iteration-len(value))])
    return old

def write_csv(filename, urls, input):
    print("Writing {}...".format(filename))
    with open(filename, 'w', newline='') as csvfile:
        out = writer(csvfile)
        out.writerow(["Test name", "#"] + urls)
        tests = sorted(input.keys())
        for test in tests:
            out.writerow([test, len([i for i in input[test] if i != ""])] + input[test])

if __name__ == "__main__":
    data = {}
    for url in urls:
        name = download(url)
        if name == "":
            print("Skipping {} due to download problem.\n".format(url))
            continue
        flakes, failures = list_problems(name)
        data[url] = (flakes, failures)
        remove(name)

    urls, flakes, failures = analyze_data(data)
    write_csv('flakes.csv', urls, flakes)
    write_csv('failures.csv', urls, failures)