evergreen_test_failures.user.js

// ==UserScript==
// @name         Show Failures by Test
// @namespace    http://tampermonkey.net/
// @version      0.1
// @description  try to take over the world!
// @author       You
// @match        https://evergreen.mongodb.com/version/*
// @grant        none
// ==/UserScript==

(function() {
    'use strict';

    function finish(pre, tests, suites) {
        var out = '';
        if (suites.length) {
            out += "BAD SUITES:";
            suites.forEach(suite => {out += '\n\t' + suite;});
            out += "\n";
        }

        if (tests.size) {
            out += "FAILING TESTS:";
            tests.forEach((tasks, name) => {
                out += '\n\t' + name;
                tasks.forEach(task => {
                    out += '\n\t\t' + task;
                });
            });
            out += "\n";
        }

        pre.text(out);
    }

    function showFailingTests() {
        button.remove();
        $('#FAILING_TESTS').remove();

        if (!version.PatchInfo) {
            alert('This only works on patch builds!');
            return;
        }

        var failingVersions = version.PatchInfo.StatusDiffs.filter(
            (obj) => obj.diff.patch.status && obj.diff.patch.status != 'success');
        var fetched = 0;

        var pre = $('<pre>', {id:'FAILING_TESTS'});
        pre.text(`LOADING...(${fetched} of ${failingVersions.length})`);
        pre.appendTo('#content');

        var suites = [];
        var tests = new Map();
        failingVersions.forEach((task)=> {
            let taskDesc = task.build_variant + ' -- ' + task.name;
            let base = $.ajax(`/rest/v1/tasks/${task.original}/status`);
            $.ajax(`/rest/v1/tasks/${task.patch}/status`).done(taskStatus => {base.done(baseStatus =>{
                fetched++;
                pre.text(`LOADING...(${fetched} of ${failingVersions.length})`);

                if ($.isEmptyObject(taskStatus.tests)) {
                    suites.push(`${taskDesc}  -- patch:${taskStatus.status}  base: ${baseStatus.status}`);
                } else {
                    for (var [test, status] of Object.entries(taskStatus.tests)) {
                        if (status.status && status.status == 'pass')
                            continue;

                        let baseInfo;
                        try {
                            baseInfo = baseStatus.tests[test].status;
                        } catch (e) {
                            console.log(e);
                            baseInfo = "???";
                        }

                        test = test.replace(/\\/g, '/');
                        test = test.replace(/\.exe$/, '');

                        if (!tests.has(test))
                            tests.set(test, []);
                        // TODO status.logs
                        // TODO base
                        tests.get(test).push(`${taskDesc} -- patch:${status.status}  base: ${baseInfo}`);
                    }
                }

                if (fetched == failingVersions.length)
                    finish(pre, tests, suites);
            });});
        });
    }

    var button = $('<button>Show Failing Tests (by test)</button>');
    button.on('click', showFailingTests);
    button.appendTo('#content');
}());