repeat a set of tasks (a configurable number of times) each with a set of commands on multiple taskcluster worker types

.taskcluster.yml

---
provisionerId: relops-3
workerType: decision
retries: 5
priority: highest
created: '2020-07-08T07:00:36.378Z'
deadline: '2020-07-08T10:00:36.378Z'
routes:
  - index.project.relops.worker-observation.decision
scopes:
  - 'queue:route:index.project.relops.worker-observation.*'
  - 'queue:scheduler-id:-'
  - 'queue:create-task:low:comm-t/t-win*'
  - 'queue:create-task:low:comm-1/b-win*'
  - 'queue:create-task:low:comm-3/b-win*'
  - 'queue:create-task:low:gecko-t/t-win*'
  - 'queue:create-task:low:gecko-1/b-win*'
  - 'queue:create-task:low:gecko-3/b-win*'
  - 'queue:create-task:low:mpd001-1/b-win*'
  - 'queue:create-task:low:mpd001-3/b-win*'
  - 'queue:create-task:low:nss-1/win*'
  - 'queue:create-task:low:nss-3/win*'
  - 'queue:create-task:low:relops-3/win*'
  - 'queue:task-group-id:taskcluster-github/*'
  - 'queue:schedule-task:taskcluster-github/*'
payload:
  image: python
  maxRunTime: 7200
  features:
    taskclusterProxy: true
  artifacts:
    public/results.json:
      path: results.json
      type: file
  command:
    - /bin/bash
    - '--login'
    - '-c'
    - >-
      python --version
      && pip install asyncio | grep -v "^[[:space:]]*$"
      && pip install pyyaml | grep -v "^[[:space:]]*$"
      && pip install slugid | grep -v "^[[:space:]]*$"
      && pip install taskcluster | grep -v "^[[:space:]]*$"
      && git clone --quiet https://gist.github.com/a2ff8966607583fbc1944fccc256a80c.git ./worker-observation
      && python ./worker-observation/decision-task.py
metadata:
  owner: grenade@mozilla.com
  source: 'https://gist.github.com/grenade/a2ff8966607583fbc1944fccc256a80c'
  name: '00 :: create-worker-tasks'
  description: create observation tasks on workers
tags: {}
extra: {}

config.yml

---
tasks:
- targets:
  - provisioner: comm-1
    workertype: b-win2012
    iterations: 2
  - provisioner: comm-3
    workertype: b-win2012
    iterations: 2
  - provisioner: comm-t
    workertype: t-win10-64
    iterations: 2
  - provisioner: comm-t
    workertype: t-win7-32
    iterations: 2
  - provisioner: gecko-1
    workertype: b-win2012
    iterations: 2
  - provisioner: gecko-3
    workertype: b-win2012
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win10-64
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win10-64-gpu-s
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win10-64-source
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win7-32
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win7-32-gpu
    iterations: 2
  - provisioner: mpd001-1
    workertype: b-win2012
    iterations: 2
  - provisioner: mpd001-3
    workertype: b-win2012
    iterations: 2
  - provisioner: nss-1
    workertype: win2012r2
    iterations: 2
  - provisioner: nss-3
    workertype: win2012r2
    iterations: 2
  - provisioner: relops-3
    workertype: win2019
    iterations: 2
  namespace: observe-software-versions
  name:
    prefix: '01 :: '
    suffix: " :: observe-software-versions"
  description:
    prefix: 'observe software versions on '
    suffix: ''
  owner: grenade@mozilla.com
  command:
  - mkdir .\public
  - chcp 65001 > nul && ver > .\public\windows-version.txt
  - chcp 65001 > nul && C:\mozilla-build\python\python.exe --version 2> .\public\python2-version.txt
  - chcp 65001 > nul && C:\mozilla-build\python3\python3.exe --version > .\public\python3-version.txt
  - >-
    chcp 65001 > nul && C:\mozilla-build\python\python.exe -c "exec(\"import
    platform;print platform.release()\")" >
    .\public\python2-platform-release.txt
  - >-
    chcp 65001 > nul && C:\mozilla-build\python3\python3.exe -c "exec(\"import
    platform;print(platform.release())\")" >
    .\public\python3-platform-release.txt
  - chcp 65001 > nul && wget.exe --version > .\public\wget-version.txt
  - chcp 65001 > nul && hg --version > .\public\hg-version.txt
  - chcp 65001 > nul && C:\generic-worker\generic-worker.exe --version > .\public\generic-worker-version.txt
  - chcp 65001 > nul && wmic os get servicepackmajorversion > .\public\servicepack-major-version.txt
  artifacts:
  - type: file
    name: public/windows-version.txt
    path: public/windows-version.txt
    regex:
      match: '\d+\.\d+\.\d+'
      group: 0
  - type: file
    name: public/python2-version.txt
    path: public/python2-version.txt
    regex:
      match: '\d+\.\d+\.\d+'
      group: 0
  - type: file
    name: public/python3-version.txt
    path: public/python3-version.txt
    regex:
      match: '\d+\.\d+\.\d+'
      group: 0
  - type: file
    name: public/python2-platform-release.txt
    path: public/python2-platform-release.txt
  - type: file
    name: public/python3-platform-release.txt
    path: public/python3-platform-release.txt
  - type: file
    name: public/wget-version.txt
    path: public/wget-version.txt
    regex:
      match: '\d+\.\d+\.\d+'
      group: 0
  - type: file
    name: public/hg-version.txt
    path: public/hg-version.txt
    regex:
      match: '\d+\.\d+\.\d+'
      group: 0
  - type: file
    name: public/generic-worker-version.txt
    path: public/generic-worker-version.txt
    regex:
      match: '\d+\.\d+\.\d+'
      group: 0
  - type: file
    name: public/servicepack-major-version.txt
    path: public/servicepack-major-version.txt
    encoding: utf-16
    line: 1
  features:
    taskclusterProxy: true
  maxruntime: 600
- targets:
  - provisioner: gecko-1
    workertype: b-win2012
    iterations: 2
  - provisioner: gecko-3
    workertype: b-win2012
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win10-64
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win10-64-gpu-s
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win10-64-source
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win7-32
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win7-32-gpu
    iterations: 2
  - provisioner: mpd001-1
    workertype: b-win2012
    iterations: 2
  - provisioner: mpd001-3
    workertype: b-win2012
    iterations: 2
  - provisioner: relops-3
    workertype: win2019
    iterations: 2
  namespace: observe-occ-config
  name:
    prefix: '02 :: '
    suffix: " :: observe-occ-config"
  description:
    prefix: 'observe occ configuration on '
    suffix: ''
  owner: grenade@mozilla.com
  command:
  - mkdir .\public
  - chcp 65001 > nul && reg query HKLM\SOFTWARE\Mozilla\OpenCloudConfig\Source /v Organisation > .\public\occ-source-organisation.txt
  - chcp 65001 > nul && reg query HKLM\SOFTWARE\Mozilla\OpenCloudConfig\Source /v Repository > .\public\occ-source-repository.txt
  - chcp 65001 > nul && reg query HKLM\SOFTWARE\Mozilla\OpenCloudConfig\Source /v Revision > .\public\occ-source-revision.txt
  artifacts:
  - type: file
    name: public/occ-source-organisation.txt
    path: public/occ-source-organisation.txt
    line: 2
    split:
      separator: "    "
      index: 2
  - type: file
    name: public/occ-source-repository.txt
    path: public/occ-source-repository.txt
    line: 2
    split:
      separator: "    "
      index: 2
  - type: file
    name: public/occ-source-revision.txt
    path: public/occ-source-revision.txt
    line: 2
    split:
      separator: "    "
      index: 2
  features:
    taskclusterProxy: true
  maxruntime: 600
- targets:
  - provisioner: gecko-t
    workertype: t-win10-64-gpu-s
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win7-32-gpu
    iterations: 2
  namespace: observe-nvidia-driver-version
  name:
    prefix: '03 :: '
    suffix: " :: observe-nvidia-driver-version"
  description:
    prefix: 'observe nvidia driver version on '
    suffix: ''
  owner: grenade@mozilla.com
  command:
  - mkdir .\public
  - chcp 65001 > nul && "C:\Program Files\NVIDIA Corporation\NVSMI\nvidia-smi" --query-gpu=driver_version --format=csv > .\public\nvidia-driver-version.csv
  - chcp 65001 > nul && "C:\Program Files\NVIDIA Corporation\NVSMI\nvidia-smi" > .\public\nvidia-driver-version.txt
  artifacts:
  - type: file
    name: public/nvidia-driver-version.csv
    path: public/nvidia-driver-version.csv
  - type: file
    name: public/nvidia-driver-version.txt
    path: public/nvidia-driver-version.txt
  features:
    taskclusterProxy: true
  maxruntime: 600
- targets:
  - provisioner: gecko-1
    workertype: b-win2012
    iterations: 2
  - provisioner: gecko-3
    workertype: b-win2012
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win10-64
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win10-64-gpu-s
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win10-64-source
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win7-32
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win7-32-gpu
    iterations: 2
  namespace: observe-chrome-version
  name:
    prefix: '04 :: '
    suffix: " :: observe-chrome-version"
  description:
    prefix: 'observe chrome version on '
    suffix: ''
  owner: grenade@mozilla.com
  command:
  - mkdir .\public
  - chcp 65001 > nul && wget -O .\get-chrome-version.cmd https://gist.githubusercontent.com/grenade/a2ff8966607583fbc1944fccc256a80c/raw/get-chrome-version.cmd
  - >-
    chcp 65001 > nul && .\get-chrome-version.cmd > .\public\chrome-version.txt
  artifacts:
  - type: file
    name: public/chrome-version.txt
    path: public/chrome-version.txt
  features:
    taskclusterProxy: true
  maxruntime: 600
- targets:
  - provisioner: comm-1
    workertype: b-win2012
    iterations: 2
  - provisioner: comm-3
    workertype: b-win2012
    iterations: 2
  - provisioner: comm-t
    workertype: t-win10-64
    iterations: 2
  - provisioner: comm-t
    workertype: t-win7-32
    iterations: 2
  - provisioner: gecko-1
    workertype: b-win2012
    iterations: 2
  - provisioner: gecko-3
    workertype: b-win2012
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win10-64
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win10-64-gpu-s
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win10-64-source
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win7-32
    iterations: 2
  - provisioner: gecko-t
    workertype: t-win7-32-gpu
    iterations: 2
  - provisioner: mpd001-1
    workertype: b-win2012
    iterations: 2
  - provisioner: mpd001-3
    workertype: b-win2012
    iterations: 2
  - provisioner: relops-3
    workertype: win2019
    iterations: 2
  namespace: observe-filesystem
  name:
    prefix: '05 :: '
    suffix: " :: observe-filesystem"
  description:
    prefix: 'observe filesystem on '
    suffix: ''
  owner: grenade@mozilla.com
  command:
  - mkdir .\public
  - 'chcp 65001 > nul && tree c: /f /a > .\public\filesystem-drive-c.txt'
  - 'chcp 65001 > nul && tree y: /f /a > .\public\filesystem-drive-y.txt'
  - 'chcp 65001 > nul && tree z: /f /a > .\public\filesystem-drive-z.txt'
  artifacts:
  - type: file
    name: public/filesystem-drive-c.txt
    path: public/filesystem-drive-c.txt
  - type: file
    name: public/filesystem-drive-y.txt
    path: public/filesystem-drive-y.txt
  - type: file
    name: public/filesystem-drive-z.txt
    path: public/filesystem-drive-z.txt
  features:
    taskclusterProxy: true
  maxruntime: 600

debug-config.yml

---
tasks:
- targets:
  - provisioner: aws-provisioner-v1
    workertype: gecko-t-win7-32-gpu
    iterations: 1
  - provisioner: aws-provisioner-v1
    workertype: gecko-3-b-win2012
    iterations: 1
  - provisioner: releng-hardware
    workertype: gecko-t-win10-64-hw-dev
    iterations: 1
  namespace: collect-software-versions
  name:
    prefix: '01 :: '
    suffix: " :: collect-software-versions"
  description:
    prefix: 'retrieve software versions on '
    suffix: ''
  owner: grenade@mozilla.com
  command:
  - mkdir .\public
  - chcp 65001 > nul && ver > .\public\windows-version.txt
  - chcp 65001 > nul && C:\mozilla-build\python\python.exe --version 2> .\public\python2-version.txt
  - chcp 65001 > nul && C:\mozilla-build\python3\python3.exe --version > .\public\python3-version.txt
  - >-
    chcp 65001 > nul && C:\mozilla-build\python\python.exe -c "exec(\"import
    platform;print platform.release()\")" >
    .\public\python2-platform-release.txt
  - >-
    chcp 65001 > nul && C:\mozilla-build\python3\python3.exe -c "exec(\"import
    platform;print(platform.release())\")" >
    .\public\python3-platform-release.txt
  - chcp 65001 > nul && wget.exe --version > .\public\wget-version.txt
  - chcp 65001 > nul && hg --version > .\public\hg-version.txt
  - chcp 65001 > nul && C:\generic-worker\generic-worker.exe --version > .\public\generic-worker-version.txt
  - chcp 65001 > nul && wmic os get servicepackmajorversion > .\public\servicepack-major-version.txt
  artifacts:
  - type: file
    name: public/windows-version.txt
    path: public/windows-version.txt
    regex:
      match: '\d+\.\d+\.\d+'
      group: 0
  - type: file
    name: public/python2-version.txt
    path: public/python2-version.txt
    regex:
      match: '\d+\.\d+\.\d+'
      group: 0
  - type: file
    name: public/python3-version.txt
    path: public/python3-version.txt
    regex:
      match: '\d+\.\d+\.\d+'
      group: 0
  - type: file
    name: public/python2-platform-release.txt
    path: public/python2-platform-release.txt
  - type: file
    name: public/python3-platform-release.txt
    path: public/python3-platform-release.txt
  - type: file
    name: public/wget-version.txt
    path: public/wget-version.txt
    regex:
      match: '\d+\.\d+\.\d+'
      group: 0
  - type: file
    name: public/hg-version.txt
    path: public/hg-version.txt
    regex:
      match: '\d+\.\d+\.\d+'
      group: 0
  - type: file
    name: public/generic-worker-version.txt
    path: public/generic-worker-version.txt
    regex:
      match: '\d+\.\d+\.\d+'
      group: 0
  - type: file
    name: public/servicepack-major-version.txt
    path: public/servicepack-major-version.txt
    encoding: utf-16
    line: 1
  features:
    taskclusterProxy: true
  maxruntime: 600
- targets:
  - provisioner: aws-provisioner-v1
    workertype: gecko-t-win7-32-gpu
    iterations: 1
  - provisioner: aws-provisioner-v1
    workertype: gecko-3-b-win2012
    iterations: 1
  namespace: collect-occ-config
  name:
    prefix: '02 :: '
    suffix: " :: collect-occ-config"
  description:
    prefix: 'determine occ configuration on '
    suffix: ''
  owner: grenade@mozilla.com
  command:
  - mkdir .\public
  - chcp 65001 > nul && reg query HKLM\SOFTWARE\Mozilla\OpenCloudConfig\Source /v Organisation > .\public\occ-source-organisation.txt
  - chcp 65001 > nul && reg query HKLM\SOFTWARE\Mozilla\OpenCloudConfig\Source /v Repository > .\public\occ-source-repository.txt
  - chcp 65001 > nul && reg query HKLM\SOFTWARE\Mozilla\OpenCloudConfig\Source /v Revision > .\public\occ-source-revision.txt
  artifacts:
  - type: file
    name: public/occ-source-organisation.txt
    path: public/occ-source-organisation.txt
    line: 2
    split:
      separator: "    "
      index: 2
  - type: file
    name: public/occ-source-repository.txt
    path: public/occ-source-repository.txt
    line: 2
    split:
      separator: "    "
      index: 2
  - type: file
    name: public/occ-source-revision.txt
    path: public/occ-source-revision.txt
    line: 2
    split:
      separator: "    "
      index: 2
  features:
    taskclusterProxy: true
  maxruntime: 600
- targets:
  - provisioner: releng-hardware
    workertype: gecko-t-win10-64-hw-dev
    iterations: 1
  namespace: collect-ronin-config
  name:
    prefix: '03 :: '
    suffix: " :: collect-ronin-config"
  description:
    prefix: 'determine ronin configuration on '
    suffix: ''
  owner: grenade@mozilla.com
  command:
  - mkdir .\public
  - chcp 65001 > nul && reg query HKLM\SOFTWARE\Mozilla\ronin_puppet\Source /v Organisation > .\public\ronin-source-organisation.txt
  - chcp 65001 > nul && reg query HKLM\SOFTWARE\Mozilla\ronin_puppet\Source /v Repository > .\public\ronin-source-repository.txt
  - chcp 65001 > nul && reg query HKLM\SOFTWARE\Mozilla\ronin_puppet\Source /v Revision > .\public\ronin-source-revision.txt
  artifacts:
  - type: file
    name: public/ronin-source-organisation.txt
    path: public/ronin-source-organisation.txt
    line: 2
    split:
      separator: "    "
      index: 2
  - type: file
    name: public/ronin-source-repository.txt
    path: public/ronin-source-repository.txt
    line: 2
    split:
      separator: "    "
      index: 2
  - type: file
    name: public/ronin-source-revision.txt
    path: public/ronin-source-revision.txt
    line: 2
    split:
      separator: "    "
      index: 2
  features:
    taskclusterProxy: true
  maxruntime: 600

decision-task.py

import asyncio
import os
import json
import re
import slugid
import taskcluster.aio
import time
import urllib
import urllib.request
import yaml
from datetime import datetime, timedelta
from gzip import decompress
from random import randint, seed

seed(1)

GIST_USER = 'grenade'
GIST_SHA = 'a2ff8966607583fbc1944fccc256a80c'


async def create_task(provisioner, workerType, taskGroupId, task, iteration, iterations):
  global asyncQueue
  taskId = slugid.nice()
  payload = {
    'created': '{}Z'.format(datetime.utcnow().isoformat()[:-3]),
    'deadline': '{}Z'.format((datetime.utcnow() + timedelta(days=3)).isoformat()[:-3]),
    'provisionerId': provisioner,
    'workerType': workerType,
    'taskGroupId': taskGroupId,
    'routes': [
      'index.project.relops.worker-observation.{}.{}.{}'.format(provisioner, workerType, task['namespace']),
      'index.project.relops.worker-observation.latest.{}.{}.{}'.format(provisioner, workerType, task['namespace']),
      'index.project.relops.worker-observation.daily.{}.{}.{}.{}'.format(datetime.utcnow().strftime("%Y%m%d"), provisioner, workerType, task['namespace']),
      'index.project.relops.worker-observation.hourly.{}.{}.{}.{}'.format(datetime.utcnow().strftime("%Y%m%d%H"), provisioner, workerType, task['namespace'])
    ],
    'scopes': [],
    'payload': {
      'maxRunTime': task['maxruntime'],
      'command': task['command'],
      'artifacts': list(map(lambda x: {'type': x['type'],'name': x['name'],'path': x['path']}, task['artifacts'])),
      'features': task['features']
    },
    'metadata': {
      'name': '{} {}/{} {} :: {}/{}'.format(task['name']['prefix'], provisioner, workerType, task['name']['suffix'], iteration, iterations) if (iterations < 10) else '{}{}/{}{} {:02d}/{}'.format(task['name']['prefix'], provisioner, workerType, task['name']['suffix'], iteration, iterations) if (iterations < 100) else '{}{}/{}{} {:03d}/{}'.format(task['name']['prefix'], provisioner, workerType, task['name']['suffix'], iteration, iterations),
      'description': '{}{}/{}{}'.format(task['description']['prefix'], provisioner, workerType, task['description']['suffix']),
      'owner': task['owner'],
      'source': 'https://gist.github.com/{}/{}'.format(GIST_USER, GIST_SHA)
    }
  }
  print('creating {}/{} task {} ({}/{} {}) ({}/groups/{}/tasks/{})'.format(provisioner, workerType, task['namespace'], iteration, iterations, taskId, os.environ.get('TASKCLUSTER_ROOT_URL'), os.environ.get('TASK_ID'), taskId))
  return await asyncQueue.createTask(taskId, payload)

  
async def print_task_artifacts(provisioner, workerType, taskGroupId, taskNamespace, task, iteration, iterations):
  global results, asyncQueue
  taskStatus = await create_task(provisioner, workerType, taskGroupId, task, iteration, iterations)
  print('{}/{} - {} ({}/{} {}): {}'.format(provisioner, workerType, taskNamespace, iteration, iterations, taskStatus['status']['taskId'], taskStatus['status']['state']))
  while taskStatus['status']['state'] not in ['completed', 'failed']:
    await asyncio.sleep(randint(10, 30) if taskStatus['status']['state'] == 'pending' else 0.5)
    print('{}/{} - {} ({}/{} {}): {}'.format(provisioner, workerType, taskNamespace, iteration, iterations, taskStatus['status']['taskId'], taskStatus['status']['state']))
    taskStatus = await asyncQueue.status(taskStatus['status']['taskId'])
  print('{}/{} - {} ({}/{} {}): {} on run {}'.format(provisioner, workerType, taskNamespace, iteration, iterations, taskStatus['status']['taskId'], taskStatus['status']['state'], taskStatus['status']['runs'][-1]['runId']))  
  for artifactDefinition in task['artifacts']:
    artifactUrl = 'https://taskcluster-artifacts.net/{}/{}/{}'.format(taskStatus['status']['taskId'], taskStatus['status']['runs'][-1]['runId'], artifactDefinition['name'])
    print('{}/{} - {} ({}/{} {}): {}'.format(provisioner, workerType, taskNamespace, iteration, iterations, taskStatus['status']['taskId'], artifactUrl))
    try:
      artifactContent = decompress(urllib.request.urlopen(urllib.request.Request(artifactUrl)).read()).decode(artifactDefinition['encoding'] if 'encoding' in artifactDefinition else 'utf-8')
      if 'file-missing-on-worker' in artifactContent:
        artifactContent = None
    except Exception as e:
      print('error fetching artifact {}'.format(artifactUrl), e)
      artifactContent = None
    if artifactContent is not None:
      artifactLine = artifactContent.split('\n')[artifactDefinition['line']].strip() if 'line' in artifactDefinition else artifactContent.strip()
      artifactText = artifactLine.split(artifactDefinition['split']['separator'])[artifactDefinition['split']['index']].strip() if 'split' in artifactDefinition else artifactLine
      if 'regex' in artifactDefinition:
        try:
          artifactText = re.search(artifactDefinition['regex']['match'], artifactText).group(artifactDefinition['regex']['group'])
        except Exception as e:
          artifactText = ''
          print('error matching regex: "{}", group: {}'.format(artifactDefinition['regex']['match'], artifactDefinition['regex']['group']), e)
    else:
      artifactText = ''
    print('{}/{} - {} ({}/{} {}): {}: {}'.format(provisioner, workerType, taskNamespace, iteration, iterations, taskStatus['status']['taskId'], artifactDefinition['name'], artifactText))
    run = taskStatus['status']['runs'][-1]['runId']
    if workerType in results:
      if taskNamespace in results[workerType]:
        if 'iteration-{}'.format(iteration) in results[workerType][taskNamespace]:
          results[workerType][taskNamespace]['iteration-{}'.format(iteration)].update({ 'task': taskStatus['status'], os.path.splitext(os.path.basename(artifactDefinition['name']))[run]: artifactText })
        else:
          results[workerType][taskNamespace].update({ 'iteration-{}'.format(iteration): { 'task': taskStatus['status'], os.path.splitext(os.path.basename(artifactDefinition['name']))[run]: artifactText } })
      else:
        results[workerType].update({ taskNamespace: { 'iteration-{}'.format(iteration): { 'task': taskStatus['status'], os.path.splitext(os.path.basename(artifactDefinition['name']))[run]: artifactText } } })
    else:
      results.update({ workerType: { taskNamespace: { 'iteration-{}'.format(iteration): { 'task': taskStatus['status'], os.path.splitext(os.path.basename(artifactDefinition['name']))[run]: artifactText } } } })


async def close(session):
  await session.close()


config = yaml.load(urllib.request.urlopen('https://gist.githubusercontent.com/{}/{}/raw/config.yml?{}'.format(GIST_USER, GIST_SHA, slugid.nice())).read())
taskclusterOptions = {
  'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']
}

start = time.time()
loop = asyncio.get_event_loop()

session = taskcluster.aio.createSession(loop=loop)
asyncQueue = taskcluster.aio.Queue(taskclusterOptions, session=session)

tasks = []
results = {}
for task in config['tasks']:
  for target in task['targets']:
    for i in range(1, (target['iterations'] + 1)):
      tasks.append(asyncio.ensure_future(print_task_artifacts(target['provisioner'], target['workertype'], os.environ.get('TASK_ID'), task['namespace'], task, i, target['iterations'])))

loop.run_until_complete(asyncio.wait(tasks, timeout=1200))
loop.run_until_complete(close(session))
loop.close()
with open('results.json', 'w') as fp:
  json.dump(results, fp, indent=2, sort_keys=True)
end = time.time()
print("total time: {}".format(end - start))

get-chrome-version.cmd

@ECHO OFF

:: Look for machine-wide Chrome installs (stable, Beta, and Dev).
:: Get the name, running version (if an update is pending relaunch), and
:: installed version of each.
FOR %%A IN (
  {8A69D345-D564-463c-AFF1-A69D9E530F96},
  {8237E44A-0054-442C-B6B6-EA0509993955},
  {401C381F-E0DE-4B85-8BD8-3F3F14FBDA57},
  {4ea16ac7-fd5a-47c3-875b-dbf4a2008c20}) DO (
  reg query HKLM\Software\Google\Update\Clients\%%A > NUL 2>&1
  if %errorlevel% equ 0 (
    reg query HKLM\Software\Google\Update\Clients\%%A /v name > NUL 2>&1
    if %errorlevel% equ 0 reg query HKLM\Software\Google\Update\Clients\%%A /v name /reg:32 2> NUL
    reg query HKLM\Software\Google\Update\Clients\%%A /v opv > NUL 2>&1
    if %errorlevel% equ 0 reg query HKLM\Software\Google\Update\Clients\%%A /v opv /reg:32 2> NUL
    reg query HKLM\Software\Google\Update\Clients\%%A /v pv > NUL 2>&1
    if %errorlevel% equ 0 reg query HKLM\Software\Google\Update\Clients\%%A /v pv /reg:32 2> NUL
  )
)

:: Look for Chrome installs in the current user's %LOCALAPPDATA% directory
:: (stable, Beta, Dev, and canary).
:: Get the name, running version (if an update is pending relaunch), and
:: installed version of each.
FOR %%A IN (
  {8A69D345-D564-463c-AFF1-A69D9E530F96},
  {8237E44A-0054-442C-B6B6-EA0509993955},
  {401C381F-E0DE-4B85-8BD8-3F3F14FBDA57},
  {4ea16ac7-fd5a-47c3-875b-dbf4a2008c20}) DO (
  reg query HKCU\Software\Google\Update\Clients\%%A > NUL 2>&1
  if %errorlevel% equ 0 (
    reg query HKCU\Software\Google\Update\Clients\%%A /v name > NUL 2>&1
    if %errorlevel% equ 0 reg query HKCU\Software\Google\Update\Clients\%%A /v name /reg:32 2> NUL
    reg query HKCU\Software\Google\Update\Clients\%%A /v opv > NUL 2>&1
    if %errorlevel% equ 0 reg query HKCU\Software\Google\Update\Clients\%%A /v opv /reg:32 2> NUL
    reg query HKCU\Software\Google\Update\Clients\%%A /v pv > NUL 2>&1
    if %errorlevel% equ 0 reg query HKCU\Software\Google\Update\Clients\%%A /v pv /reg:32 2> NUL
  )
)
EXIT 0

Usage

• fork this gist.
• replace references to my username grenade, with your own github username.
• replace references to this gist id a2ff8966607583fbc1944fccc256a80c with your own gist id.
• replace references to this gists index index.project.relops.worker-observation with your custom index eg: index.project.my-custom-project.some-interesting-discovery.
• modify config.yml with suitable iterations, task names, descriptions, commands and artifacts for your needs.
• copy & paste the contents of .taskcluster.yml into the task creator and run the task.

this implementation is deprecated and has moved to https://github.com/mozilla-platform-ops/worker-observer