Proof of concept: sporadic failure finder

scanner.d

import std.stdio;
import std.net.curl : get;
import std.array : split;
import std.range;
import std.conv : to;
import std.regex;
import std.ascii;
import std.parallelism : parallel;
import std.string : strip, lineSplitter;
import std.algorithm.searching : findSplit;

enum Branch
{
    master = 1,
    stable = 14
}
enum Repo
{
    dmd = 1,
    druntime = 2,
    phobos = 3
}
enum Step
{
    testing_DMD = 5,
    testing_Druntime = 6,
    testing_Phobos = 7
}
enum Platform
{
    Darwin_64_32 = 0,
    Darwin_64_64,
    FreeBSD_32,
    FreeBSD_64,
    Linux_32,
    Linux_64_32,
    Linux_64_64,
    Win_32,
    Win_64_32,
}
struct Fail
{
    Branch branch;
    bool isPull;
    Repo repo; // for pulls
    uint pullid; // for pulls
    Step step;
    Platform platform;
    uint runid;
    string log;
    bool failed;
    int errorcode;
    string errorstep;
}
struct GFail
{
    Fail*[] fs;
}
auto baseUrl = "https://auto-tester.puremagic.com/";
static re = ctRegex!(`[gMm]{1,2}ake[^:]*:[^\[]+\[(?P<step>[^\]]+)\]\s+[Ee]rror\s+(?P<code>[0-9]+)`);

void main(string[] args)
{
    if (args.length < 2)
        return;
    Fail[] fs;
    foreach (l; File(args[1]).byLine)
    {
        auto sl = l.split;
        if (!sl.length)
            break;
        Fail f;
        f.branch = sl[0].to!int.to!Branch;
        sl.popFront();
        f.isPull = sl.length > 4;
        if (f.isPull)
        {
            f.repo = sl[0].to!int.to!Repo;
            sl.popFront();
            f.pullid = sl[0].to!int;
            sl.popFront();
        }
        f.step = sl[0].to!int.to!Step;
        f.platform = sl[1].to!int.to!Platform;
        f.runid = sl[2].to!int;
        f.log = sl[3].idup;
        fs ~= f;
    }
    foreach (ref f; fs.parallel)
    {
        const(char)[] log;
        try
        {
            log = get(baseUrl~f.log);
        }
        catch (Exception e)
        {
            f.failed = true;
            continue;
        }
        bool foundMatch;
        foreach (line; log.lineSplitter)
        {
            auto matches = line.matchFirst(re);
            if (!matches)
                continue;
            f.errorcode = matches["code"].to!int;
            f.errorstep = matches["step"].idup;
            auto tmp = f.errorstep.findSplit("/linux");
            f.errorstep = tmp[0] ~ tmp[2];
            tmp = f.errorstep.findSplit("/osx");
            f.errorstep = tmp[0] ~ tmp[2];
            tmp = f.errorstep.findSplit("/win");
            f.errorstep = tmp[0] ~ tmp[2];
            tmp = f.errorstep.findSplit("32");
            f.errorstep = tmp[0] ~ tmp[2];
            tmp = f.errorstep.findSplit("64");
            f.errorstep = tmp[0] ~ tmp[2];
            foundMatch = true;
            break;
        }
        f.failed = !foundMatch;
    }
    GFail[string][Step] groups;
    GFail[string] wrkarnd;
    groups[Step.testing_DMD] = wrkarnd.dup;
    groups[Step.testing_Druntime] = wrkarnd.dup;
    groups[Step.testing_Phobos] = wrkarnd.dup;

    foreach (ref f; fs)
    {
        if (f.failed || !f.errorstep.length)
            continue;
        auto p = f.errorstep in groups[f.step];
        if (p)
        {
            p.fs ~= &f;
        }
        else
        {
            groups[f.step][f.errorstep] = GFail([&f]);
        }
    }

    foreach (step, sfails; groups)
    {
        writeln("Failed ",step,":");
        foreach (sfail; sfails)
        {
            writeln("    ",sfail.fs[0].errorstep);
            foreach (f; sfail.fs)
            {
                write("        ");
                write(f.branch," ");
                if (f.isPull)
                {
                    write(f.repo,"/",f.pullid,"    ",f.platform);
                }
                else
                    write(f.platform);
                writeln("  ",f.runid);
            }
        }
        writeln();
    }
}

scraper.py

from bs4 import BeautifulSoup
from urllib.request import urlopen
import re

platforms = ['Darwin_64_32','Darwin_64_64', 'FreeBSD_32', 'FreeBSD_64',
             'Linux_32', 'Linux_64_32', 'Linux_64_64',
             'Win_32', 'Win_64_32']
repos = ['dmd','druntime','phobos']

def parseTestRun(tr):
    testrun = {}
    testrun['id'] = tr.find("td", class_="id").contents[0].contents[0]
    platform = tr.find("td", class_="platform").contents[0]
    testrun['platform'] = platform.replace("_32_64","_64_32")
    #strtime = tr.find("td", class_="starttime").contents[0]
    #testrun['starttime'] = strptime(strtime, "%Y-%m-%d %H:%M:%S")
    testrun['result'] = tr.find("td", class_="result").contents[0]
    testrun['sha'] = tr.find("td", class_="sha").contents[0]
    return testrun

def extractFailures(link):
    repoid = int(re.search("repoid=([0-9]+)", link).group(1))
    pullid = int(re.search("pullid=([0-9]+)", link).group(1))
    r = urlopen('https://auto-tester.puremagic.com/'+link).read()
    soup = BeautifulSoup(r, "html.parser")
    rawtrs = soup.find_all("tr", class_=re.compile("pulltestrun.*"))
    prrevs = {}
    if len(rawtrs) == 0:
        return []

    for rawtr in rawtrs:
        testrun = parseTestRun(rawtr)
        if not testrun['sha'] in prrevs:
            prrevs[testrun['sha']] = [testrun]
        else:
            prrevs[testrun['sha']].append(testrun)

    deleteme = []    
    for trset in list(prrevs.keys()):
        failstats = {}
        for tr in prrevs[trset]:
            if tr['platform'] in failstats:
                (runs,fails) = failstats[tr['platform']]
                failstats[tr['platform']] = (runs + 1, fails + (1 if tr['result'] == 'F' else 0))
            else:
                failstats[tr['platform']] = (1, 1 if tr['result'] == 'F' else 0)
        for stats in failstats:
            (runs,fails) = failstats[stats]
            if fails / runs > 0.90:
                deleteme.append(trset)
                break

    for d in deleteme:
        del prrevs[d]

    singleFails = []
    for trset in prrevs:
        overlap = {}
        runset = {}
        for tr in prrevs[trset]:
            if tr['platform'] in runset:
                if not tr['platform'] in overlap:
                    overlap[tr['platform']] = [tr]
                else:
                    overlap[tr['platform']].append(tr)
            else:
                runset[tr['platform']] = tr
                if len(runset) == len(platforms):
                    singleFail = None
                    for r in runset:
                        if runset[r]['result'] == 'F':
                            if singleFail is None:
                                singleFail = runset[r]
                            else:
                                singleFail = None
                                break
                    if not singleFail is None:
                        singleFails.append(singleFail)
                    runset = {}
                    for k in list(overlap.keys()):
                        runset[k] = overlap[k].pop()
                        if len(overlap[k]) == 0:
                            del overlap[k]
        #print (runset.keys())

    for sf in singleFails:
        sf['repoid'] = repoid
        sf['pullid'] = pullid

    return singleFails

def processFailure(fail,projectid,ispull):
    runid = fail['id']
    isPull = '&isPull=true' if ispull else ''
    r = urlopen('https://auto-tester.puremagic.com/show-run.ghtml?'+projectid+'&runid='+runid+isPull).read()
    soup = BeautifulSoup(r, "html.parser")
    steps = soup.find_all("td", class_=re.compile("testcell.+"))
    #if soup.find('iframe') is None:
        #print (projectid+'&runid='+runid+isPull)
    loglink = soup.find('iframe')['src']
    failstep = 0
    for step in steps:
        if 'pass' in step['class']:
            failstep += 1
    if not ispull:
        failstep += 1 # because there's no merge step
    fail['failstep'] = failstep
    fail['log'] = loglink
    return

def scanProject(project):
    projectid = 'projectid='+str(project)
    mainfails = []
    for platform in platforms + ['Linux_32_64', 'Win_32_64']:
        r = urlopen('https://auto-tester.puremagic.com/platform-history.ghtml?'+projectid+'&os='+platform).read()
        soup = BeautifulSoup(r, "html.parser")
        pfailRuns = []
        for fail in soup.find_all("td", class_="result fail"):
            run = fail.find("a", href=re.compile("show-run\.ghtml.*"))['href']
            runid = re.search("runid=([0-9]+)", run).group(1)
            f = {'id':runid, 'platform':platform.replace("_32_64","_64_32")}
            processFailure(f,projectid,False)
            if f['failstep'] >= 5:
                print(project,f['failstep'],platforms.index(f['platform']),f['id'],f['log'])
            pfailRuns.append(f)
        mainfails += pfailRuns


    r = urlopen('https://auto-tester.puremagic.com/pulls.ghtml?'+projectid).read()
    soup = BeautifulSoup(r, "html.parser")
    prfails = []
    for pull in soup.find_all("tr", class_=re.compile("pullrow.*")):
        fails = pull.find_all("td", class_=re.compile("pullcell fail.*"))
        if len(fails) > 1:
            continue
        link = pull.find("a", href=re.compile("pull-history\.ghtml.*"))['href']
        fs = extractFailures(link)
        prfails += fs
        for f in fs:
            processFailure(f,projectid,True)
            if f['failstep'] >= 5:
                print(project,f['repoid'],f['pullid'],f['failstep'],platforms.index(f['platform']),f['id'],f['log'])


    return

scanProject(1)
scanProject(14)