TaskCluster: Launching tasks locally

README.md

Launching tasks

This is a script that I use to emulate the workflow of launching a task from a hook registered in the Hook Manager on tools.taskcluster.net. By default, this will launch a task definition that will start a fennec nightly.

A hook will start a start a task using the current head of a repository. This is a painful process to work locally since you have to edit the task definition to get the correct revision for every launch.

Running

This script will allow you to generate a task that refers to a revision that you push to try.

First set up a client in the Client Manager. This client should be able to generate create the task that you specify. For nightly fennec, refer to the scopes under the hook-id:releng/nightly-fennec-dev role.

Replace the credentials in the launch script. Replace the TASK_PATH and REPO_PATH if neccesary.

Using a virtual environment, install the dependencies from the requirements.

$ virtualenv -p python3 venv
(venv)$ pip install -r requirements.txt

Before you run the script, you should change to the head that you want launch the task with. In your repository directory:

$ hg push -r . try -f

--force is necessary if your head is on a local bookmark.

Then run the launch script. You can optionally specify a revision using --rev.

(venv)$ python launch.py

launch.py

import argparse
import arrow
import json
import os
import re
import slugid
import subprocess
import sys
import taskcluster


config = {
    'credentials': {
        'clientId': 'mozilla-ldap/<your-name>@mozilla.com/<client-name>',
        'accessToken': '...'
    }
}

# Task definition to launch
TASK_PATH = 'nightly-fennec.json'
# Path to the location of a hg repository
REPO_PATH = '/Users/amiyaguchi/Work/mozilla-unified'


def main(args):
    """ Launch a templated version of the task definition. """

    parser = argparse.ArgumentParser()
    parser.add_argument('--dry-run', action='store_true')
    parser.add_argument('--rev')
    parser.add_argument('--verbose', action='store_true')
    args = parser.parse_args(args)

    # create the payload
    payload = create_definition(TASK_PATH, args)

    if args.verbose:
        # Dump the templated json body for debugging
        print(json.dumps(payload, sort_keys=True, indent=2))

    taskid = slugid.nice().decode()

    if not args.dry_run:
        queue = taskcluster.Queue(config)
        status = queue.createTask(taskid, payload)
        print('https://tools.taskcluster.net/task-inspector/#{}'.format(taskid))


def get_revision(path):
    """ Find the revision of the current head of a repository located at the path. """
    revision = subprocess.check_output(
        ['hg', 'log', '--rev', '.', '--template', '{node}'],
        cwd=path).decode()

    assert re.match('^[a-f0-9]{40}$', revision)
    return revision


def create_definition(path, args):
    """ Take a task definition and prep it for createTask. """
    if not os.path.isfile(path):
        print('path is invalid')
        sys.exit(1)

    with open(path) as f:
        contents = f.read()

    body = json.loads(contents)

    # update the current timestamps
    attrs = {
        'created': str(arrow.utcnow()),
        'deadline': str(arrow.utcnow().replace(hours=+6)),
        'expires': str(arrow.utcnow().replace(days=+7)),
    }
    body.update(attrs)

    # specify which revision to use
    if args.rev:
        revision = args.rev
    else:
        revision = get_revision(REPO_PATH)

    print("using revision hash: ", revision)
    body['payload']['env']['GECKO_HEAD_REV'] = revision
    body['payload']['command'][-1] = body['payload']['command'][-1].format(pushdate=arrow.utcnow().timestamp)

    return body


if __name__ == '__main__':
    main(sys.argv[1:])

nightly-fennec.json

{
  "provisionerId": "aws-provisioner-v1",
  "workerType": "gecko-decision",
  "payload": {
    "env": {
      "GECKO_BASE_REPOSITORY": "https://hg.mozilla.org/mozilla-central",
      "GECKO_HEAD_REPOSITORY": "https://hg.mozilla.org/try/",
      "GECKO_HEAD_REF": "tip",
      "GECKO_HEAD_REV": ""
    },
    "cache": {
      "level-1-hg-shared": "/home/worker/hg-shared",
      "level-1-checkouts": "/home/worker/checkouts"
    },
    "features": {
      "taskclusterProxy": true
    },
    "image": "taskcluster/decision:0.1.5",
    "maxRunTime": 1800,
    "command": [
      "/home/worker/bin/run-task",
      "--vcs-checkout=/home/worker/checkouts/gecko",
      "--",
      "bash",
      "-cx",
      "cd /home/worker/checkouts/gecko && ln -s /home/worker/artifacts artifacts && ./mach --log-no-times taskgraph decision --pushlog-id='0' --pushdate='0' --project='try' --message='try: -b o -p foo -u none -t none' --owner='amiyaguchi@mozilla.com' --level='1' --base-repository=$GECKO_BASE_REPOSITORY --head-repository=$GECKO_HEAD_REPOSITORY --head-ref=$GECKO_HEAD_REF --head-rev=$GECKO_HEAD_REV --revision-hash=$GECKO_HEAD_REV --triggered-by='nightly' --target-tasks-method='nightly_fennec'\n"
    ],
    "artifacts": {
      "public": {
        "type": "directory",
        "path": "/home/worker/artifacts"
      },
      "public/docker_image_contexts": {
        "type": "directory",
        "path": "/home/worker/docker_image_contexts"
      }
    }
  },
  "metadata": {
    "name": "Nightly Fennec development hook",
    "description": "The task that creates all of the other tasks in the task graph\n",
    "owner": "amiyaguchi@mozilla.com",
    "source": "https://tools.taskcluster.net/hooks/"
  },
  "tags": {
    "createdForUser": "amiyaguchi@mozilla.com"
  },
  "schedulerId": "-",
  "routes": [],
  "priority": "normal",
  "retries": 5,
  "scopes": [
      "docker-worker:cache:level-1-*",
      "docker-worker:cache:tooltool-cache",
      "scheduler:extend-task-graph:*",
      "assume:hook-id:releng/nightly-fennec-dev"
  ],
  "extra": {
    "treeherder": {
      "symbol": "D"
    }
  }
}

requirements.txt

aiohttp==0.22.5
arrow==0.8.0
chardet==2.3.0
mohawk==0.3.3
multidict==1.2.2
python-dateutil==2.5.3
requests==2.11.1
six==1.10.0
slugid==1.0.7
taskcluster==0.3.4
wheel==0.24.0