Note that some of the environment variable names have been changed and new ones introduced in case we want to easily change the metric for regression detection or the mathematical function in the future.

compare-results.js

#! /usr/bin/env node
'use strict';

var Promise = require('promise');
var fs = require('fs');
var stats = require('stats-lite');
var exec = require('child_process').exec;
require('console.table');

var ENV = {};
var SUCCESS = 0,
var REGRESSION = 1,
var FATAL = 2;

[
  'RAPTOR_REGRESSION_METRIC', // coldlaunch.visuallyLoaded
  'RAPTOR_REGRESSION_COMPARATOR', // p95
  'RAPTOR_REGRESSION_THRESHOLD', // 15
  'RAPTOR_COMPLETED_BRANCH',
  'RAPTOR_COMPLETED_PATCH',
  'RAPTOR_LOG_BRANCH', // ./raptor1.log
  'RAPTOR_LOG_PATCH' // ./raptor2.log
].forEach(function(env) {
  var value = process.env[env];

  if (!value) {
    console.log('Must provide %s environment variable to continue', env);
    return process.exit(FATAL);
  }

  ENV[env] = value;
});

var downloadLog = function(taskId, destFile) {
  return new Promise(function(resolve, reject) {
    var src = 'https://queue.taskcluster.net/v1/task/' + taskId + '/artifacts/public/logs/raptor.log';
    var cmd = 'wget -t 3 -o ' + destFile + '  ' + src;
    
    console.log('Downloading raptor.log for task', taskId);
    console.log(cmd);

    exec(cmd, function(err, stdout, stderr) {
      if (err) {
        console.log('ERROR:', error);
        return reject();
      }

      console.log('Download complete');
      resolve();
    });
  });
};

var readLog = function(raptorLog) {
  return new Promise(function(resolve, reject) {
    console.log('\nProcessing results in', raptorLog);

    fs.readFile(raptorLog, { encoding: 'utf8' }, function(err, content) {
      var data = content
        .split('\n')
        .map(JSON.parse);

      if (err) {
        reject(err);
      } else {
        resolve(data);
      }
    });
  });
}

// From raptor phase.logStats
var getStat = function(data) {
  return new Promise(function(resolve, reject) {
    var results = {};
    var metrics = [];
    var regressionResult = null;

    data.forEach(function(run) {
      Object
        .keys(run)
        .forEach(function(key) {
          if (!results[key]) {
            results[key] = [];
          }

          var entry = run[key][0];
          var value = entry.value;

          if (entry.entryType === 'memory') {
            value = value / 1024 / 1024;
          }

          results[key].push(value);
        });
    });

    Object
      .keys(results)
      .forEach(function(key) {
        var values = results[key];
        var percentile = stats.percentile(values, 0.95);

        if (key !== 'events') {
          var metric = {
            Metric: key,
            Mean: stats.mean(values).toFixed(3),
            Median: stats.median(values).toFixed(3),
            Min: Math.min.apply(Math, values).toFixed(3),
            Max: Math.max.apply(Math, values).toFixed(3),
            StdDev: stats.stdev(values).toFixed(3),
            p95: percentile ? percentile.toFixed(3) : null
          };

          metrics.push(metric);

          if (key === ENV.RAPTOR_REGRESSION_METRIC) {
            regressionResult = metric.[ENV.RAPTOR_REGRESSION_METRIC];
          }
        };
      });

    console.log('');
    console.table(metrics);
    resolve(regressionResult);
  });
};

Promise.all([
  downloadLog(ENV.RAPTOR_COMPLETED_BRANCH, ENV.RAPTOR_LOG_BRANCH),
  downloadLog(ENV.RAPTOR_COMPLETED_PATCH, ENV.RAPTOR_LOG_PATCH)
])
.then(function() {
  return Promise.all([
    readLog(ENV.RAPTOR_LOG_BRANCH),
    readLog(ENV.RAPTOR_LOG_PATCH)
  ]);
})
.then(function(logs) {
  return Promise.all([
    getStat(logs[0]),
    getStat(logs[1])
  ]);
})
.then(function(results) {
  var resultBranch = results[0];
  var resultPatch = results[1];

  console.log('%s for base branch task: %d', ENV.RAPTOR_REGRESSION_COMPARATOR, resultBranch);
  console.log('%s for patch task: %d', ENV.RAPTOR_REGRESSION_COMPARATOR, resultPatch);

  if (resultBranch === null || resultPatch === null) {
    console.log('Received invalid value for branch or patch');
    return process.exit(FATAL);
  }

  if (resultPatch <= resultBranch) {
    console.log('No regression detected, %s value has not increased,', ENV.RAPTOR_REGRESSION_COMPARATOR);
    return process.exit(SUCCESS);
  }
  
  var increase = ((resultPatch - resultBranch) / resultBranch) * 100;
  var fail = increase >= ENV.RAPTOR_REGRESSION_THRESHOLD;
  var aboveOrWithin = fail ? 'above' : 'within';

  console.log('%s has regressed %s by %d, which is %s %d%% threshold',
    ENV.RAPTOR_REGRESSION_METRIC,
    ENV.RAPTOR_REGRESSION_COMPARATOR,
    increase,
    aboveOrWithin,
    ENV.RAPTOR_REGRESSION_THRESHOLD);

  process.exit(fail ? REGRESSION : SUCCESS);
})
.catch(function(err) {
  console.log('ERROR:', err);
  process.exit(FATAL);
});