Finding Old Try Artifacts (list of artifacts: https://keybase.pub/imbstack/artifacts.gz)

index.js

require('babel-register');
let script = require('./script');
let fs = require('fs');
let _ = require('lodash');

// Each of these "sections" were run in order with previous ones being commented out
// I know this is super gross, but that's how my brain works!
// The script code is in the next file here.
// The "completed" variables are uninmportant other than allowing me to start again from where I left off
// when something breaks. They are populated by redirecting error output to a log file.
// The list of artifacts linked above is a list of (taskId, runId, storageMedium, expires, path)
// Unfortunately we don't expose "created" times for the artifacts, so I'm thinking we'll just subtract a year from expires
// and if that is more than 28 days ago we delete.

// tee this to all-builds.txt
script.findRevisions();

// tee this to all-indexed-tasks.txt
var revisions = fs.readFileSync('all-builds.txt').toString().split('\n');
script.findBuilds(revisions);

// tee this to all-tasks.txt
var decisions = fs.readFileSync('all-indexed-tasks.txt').toString().split('\n');
var completed = fs.readFileSync('completed.log').toString().split('\n').map(function(l) {return l.split('\t')[0].trim();});;
script.findTasks(_.difference(decisions, completed));

// In between "all-indexed-tasks.txt" and here, I just uniq'd all of the tasks so far to get "all-tasks-unique.txt"
var tasks = fs.readFileSync('all-tasks-unique.txt').toString().split('\n');
var completed = fs.readFileSync('completed-tasks.log').toString().split('\n').map(function(l) {return l.split('\t')[0].trim();});;
script.findArtifacts(_.difference(tasks, completed));

script.js

let taskcluster = require('taskcluster-client');
let Promise = require('bluebird');

let index = new taskcluster.Index({
  agent: {
    keepalive: true,
  }
});

let queue = new taskcluster.Queue({
  agent: {
    keepalive: true,
  }
});

let scheduler = new taskcluster.Scheduler({
  agent: {
    keepalive: true,
  }
});

let findRevisions = async () => {
  let continuationToken = '';
  do {
    let stuff = await index.listNamespaces('gecko.v2.try.revision', continuationToken ? {continuationToken} : {});
    stuff.namespaces.map((ns) => {console.log(ns.namespace);});
    continuationToken = stuff.continuationToken;
  } while (continuationToken);
};

let findBuilds = async (revisions) => {
  return Promise.map(revisions, (revision) => {
    if (!revision) {return}
    return Promise.map(['firefox', 'b2g', 'mobile', 'lint'], async (type) => {
      let ns = revision + '.' + type;
      let stuff = await index.listTasks(ns, {});
      if (stuff.tasks.length !== 0) {
        stuff.tasks.map((t) => {console.log(t.taskId)});
      }
    });
  }, {concurrency: 50});
};

let findTasks = async (decisions) => {
  return Promise.map(decisions, async (decision) => {
    // All of these will either map to a task-group or a task-graph and from
    // there we can find all of the tasks in the group/graph.
    if (!decision) {return}
    let continuationToken = null;
    let group = '';
    let graph = '';
    let taskGroupId = '';
    try {
      taskGroupId = (await queue.task(decision)).taskGroupId;
    } catch (e) {
      // The index is never cleaned up, so let's just continue
      if (e.code && e.code === 'ResourceNotFound') {
        console.error(decision + '\t\t\texpired');
        return;
      } else {
        console.error('error on ' + decision);
        throw e;
      }
    }
    try {
      // Unfortunately, we haven't always had the decision-taskId === task-group-id convention
      // We don't use continuations here because there aren't task groups with more than 1000 tasks for try
      let stuff = await queue.listTaskGroup(decision, {});
      stuff.tasks.map((task) => {console.log(task.status.taskId);});
      group = stuff.tasks.length;
    } catch (e) {
      // The index is never cleaned up, so let's just continue
      if (e.code && e.code === 'ResourceNotFound') {
      } else {
        console.error('error on ' + decision);
        throw e;
      }
    }
    try {
      let stuff = await scheduler.inspect(taskGroupId);
      stuff.tasks.map((task) => {console.log(task.taskId);});
      graph = stuff.tasks.length;
    } catch (e) {
      // The index is never cleaned up, so let's just continue
      if (e.body && e.body.message === 'task-graph not found!') {
      } else {
        console.error('error on ' + decision);
        throw e;
      }
    }
    console.error(decision, '\t', group, '\t', graph, '\t');
  }, {concurrency: 50});
};

let findArtifacts = async (tasks) => {
  return Promise.map(tasks, async (taskId) => {
    if (!taskId) {return}
    let task = null;
    try {
      task = (await queue.status(taskId)).status;
    } catch (e) {
      // Some things will have expired since the last run.
      if (e.code && e.code === 'ResourceNotFound') {
        console.error(taskId + '\texpired');
        return;
      } else {
        console.error('error on ' + task);
        throw e;
      }
    }
    await Promise.map(task.runs, async run => {
      let artifacts = await queue.listArtifacts(taskId, run.runId);
      artifacts.artifacts.map(artifact => {
        console.log(taskId, run.runId, artifact.storageType, artifact.expires, artifact.name);
      });
    });
    console.error(taskId);
  }, {concurrency: 50});
};

module.exports = {findRevisions, findBuilds, findTasks, findArtifacts};