Script to download all the logs for all the tasks in a given TaskCluster Group ID

download_tc_logs.py

#!/usr/bin/env python3

import os
import json
import requests
import argparse

from multiprocessing import Pool
from multiprocessing.dummy import Pool as ThreadPool 

BASEURL = "https://firefox-ci-tc.services.mozilla.com/api/queue/v1"
OUTPUT_DIR = "."
THREADS = 6

def getLog(filename, taskid, runid):
	if os.path.exists(filename):
		return
	l = requests.get(BASEURL + "/task/" + taskid + "/runs/" + str(runid) + "/artifacts/public/logs/live_backing.log", stream = True)
	with open(filename, 'wb') as handle:
		for block in l.iter_content(2048):
			handle.write(block)


def processTask(t):
	print("Downloading", len(t['status']['runs']), "runs for task", t['status']['taskId'], " (", t['taskIndex'], "/", str(len(j['tasks'])), ")")
	for r in t['status']['runs']:
		prefix = t['task']['metadata']['name'].replace("/", "_")
		taskid = t['status']['taskId']
		runid = r['runId']
		filename = os.path.join(OUTPUT_DIR, prefix + "_" + taskid + "_" + str(runid) + ".log")
		getLog(filename, taskid, runid)

def processTaskGroup(j, use_threads):
	i = 0
	for t in j['tasks']:
		t['taskIndex'] = i
		i += 1

	if use_threads:
		pool = ThreadPool(THREADS) 
		pool.map(processTask, j['tasks'])
		pool.close()
		pool.join()
	else:
		for t in j['tasks']:
			processTask(t)


if __name__ == "__main__":
	parser = argparse.ArgumentParser(description='Download all logs from a TC run')
	parser.add_argument('taskgroupid', metavar='TG', type=str, help='Taskgroup ID')
	parser.add_argument('--no-threads', action="store_true", help='Do not use multithreading to find commits.')
	args = parser.parse_args()

	header = {'Accept': 'application/json'}
	r = requests.get(BASEURL + "/task-group/" + args.taskgroupid + "/list", headers=header)
	j = json.loads(r.text)
	processTaskGroup(j, not args.no_threads)
	while 'continuationToken' in j:
		r = requests.get(BASEURL + "/task-group/" + args.taskgroupid + "/list?continuationToken=" + j['continuationToken'])
		j = json.loads(r.text)

	processTaskGroup(j, not args.no_threads)