Bug 1281004 example 2

CHANGED

This creates expliict tierN lists for all-tests

It also implements a new $inherit magic attribute in the test definition (tests.yml). This magic attribute
is *only* allowed one deep, so no 30 levels of inheritence. All attributes in the parent test can be overridden
by the "child" version.

It has broader potential uses than the tier example here, but could become a tangled mess of annoyance.

test-platforms.yml

# This file maps build platforms to test platforms.  In some cases, a
# single build may be tested on multiple test platforms, but a single test
# platform can only link to one build platform.  Both bulid and test platforms
# are represented as <platform>/<type>, where <type> is what Treeherder calls a
# collection.
#
# Each test platform further specifies the set of tests that will be scheduled
# for the platform, referring to tests defined in test-sets.yml.
#
# Note that set does not depend on the tree; tree-dependent job selection
# should be performed in the target task selection phase of task-grpah
# generation.
linux64/debug:
    build-platform: linux64/debug
    test-set: all-tests
linux64/opt:
    build-platform: linux64/opt
    test-set: all-tests-tier2
linux64/pgo:
    build-platform: linux64-pgo/opt
    test-set: all-tests-tier2
#mulet/opt:
#    build-platform: mulet/opt
#    test-set: mulet-tests

test-sets.yml

# Each key in this file specifies a set of tests to run.  Different test sets
# may, for example, be bound to different test platforms.
#
# Note that set does not depend on the tree; tree-dependent job selection
# should be performed in the target task selection phase of task-grpah
# generation.
#
# A test set has a name, and a list of tests that it contains.   Other test
# sets can be included with `include: ..`.  The result is treated as a set,
# with any duplicate tests discarded.
#
# Test names given here reference tests.yml.

all-tests:
    #- include: mochitests
    #- include: reftests
    #- include: web-platform-tests
    - cppunit
    - crashtest
    - crashtest-e10s
    #- external-media-tests
    #- firefox-ui-functional
    #- firefox-ui-functional-e10s
    #- gtest
    #- jittests
    #- jsreftest
    #- jsreftest-e10s
    #- marionette
    #- marionette-e10s
    #- xpcshell

all-tests-tier2:
    #- include: mochitests
    #- include: reftests
    #- include: web-platform-tests
    - cppunit-tier2
    #- ...

mulet-tests:
    - reftest
    - mochitest
    # TODO: these only run on m-c (filter in target task set)
    - gaia-build
    - gaia-build-unit
    - gaia-js-integration
    - gaia-linter
    - gaia-unit
    - gaia-unit-oop

mochitests:
    - mochitest
    - mochitest-e10s
    - mochitest-a11y
    - mochitest-browser-chrome
    - mochitest-browser-chrome-e10s
    - mochitest-chrome
    - mochitest-devtools-chrome
    - mochitest-devtools-chrome-e10s
    - mochitest-jetpack
    - mochitest-media
    - mochitest-media-e10s
    - mochitest-webgl
    - mochitest-webgl-e10s
    - mochitest-gpu
    - mochitest-gpu-e10s
    - mochitest-clipboard
    - mochitest-clipboard-e10s

reftests:
    - reftest
    - reftest-e10s
    - reftest-no-accel
    - reftest-no-accel-e10s

web-platform-tests:
    - web-platform-tests
    - web-platform-tests-e10s
    - web-platform-tests-reftests
    - web-platform-tests-reftests-e10s

tests.yml

# Each stanza here describes a particular test.  This information is used by
# the TestTask class to generate a task description.  Attributes of a test are:
#
#   description: description of the suite, for the task metadata
#
#   suite: test suite name, or <suite>/<flavor>
#
#   aliases: other names, in addition to the name of the stanza, that can be
#   used to select this test in try syntax
#
#   treeherder-symbol: the symbol, or group(symbol), under which this task
#   should appear in treeherder.
#
#   worker-type: the provisioner-id/worker-type for the task
#
#   mozharness-script: the mozharness script used to run this task
#
#   mozharness-no-read-buildbot-config: set to true if the mozharness script
#   also runs in Buildbot and tries to read a buildbot config file
#
#   mozharness-config: the config file or (space-separated) files required for
#   the task
#
#   mozharness-extra-options: additional command-line options for mozharness,
#   beyond those automatically added.
#
#   tier: the sheriffing tier for this task (default: 1)
#
#   chunks: number of chunks to create for this task (default: 1)
#
#   max-run-time: seconds of runtime after which the task will be killed
#   (default 3600)
#
# As a special case, the `$include: testname` syntax will include another test
# definition, with the locally defined values merged into it.  Use this
# sparingly, and in a fashion that remains readable!  For example, a test that
# adds a single parameter such as --e10s to another test is a good use of
# $include, if the e10s test description immediately follows the included test.
# Recursive includes are not supported.

cppunit:
    description: "CPP Unit Tests"
    suite: cppunittest
    aliases: [TODO]
    treeherder-symbol: tc(Cpp)
    worker-type: aws-provisioner-v1/desktop-test
    mozharness-script: mozharness/scripts/desktop_unittest.py
    mozharness-no-read-buildbot-config: true
    mozharness-config: >
        mozharness/configs/unittests/linux_unittest.py
        mozharness/configs/remove_executables.py
    mozharness-extra-options:
        - --cppunittest-suite=cppunittest

cppunit-tier2:
    $inherit: cppunit
    tier: 2

crashtest:
    description: "Crashtest run"
    suite: reftest/crashtest
    aliases: [TODO]
    treeherder-symbol: tc-R(C)
    worker-type: aws-provisioner-v1/desktop-test
    mozharness-script: mozharness/scripts/desktop_unittest.py
    mozharness-no-read-buildbot-config: true
    mozharness-config: >
        mozharness/configs/unittests/linux_unittest.py
        mozharness/configs/remove_executables.py
    mozharness-extra-options:
        - --reftest-suite=crashtest

crashtest-e10s:
    $include: crashtest
    treeherder-symbol: tc-R-e10s(C)
    mozharness-extra-options:
        - --e10s

#external-media-tests:
#firefox-ui-functional:
#firefox-ui-functional-e10s:
#gaia-build:
#gaia-build-unit:
#gaia-js-integration:
#gaia-linter:
#gaia-unit:
#gaia-unit-oop:
#gtest:
#jittests:
#jsreftest:
#jsreftest-e10s:
#marionette:
#marionette-e10s:
#mochitest:
#mochitest-a11y:
#mochitest-browser-chrome:
#mochitest-browser-chrome-e10s:
#mochitest-chrome:
#mochitest-clipboard:
#mochitest-clipboard-e10s:
#mochitest-devtools-chrome:
#mochitest-devtools-chrome-e10s:
#mochitest-e10s:
#mochitest-gpu:
#mochitest-gpu-e10s:
#mochitest-jetpack:
#mochitest-media:
#mochitest-media-e10s:
#mochitest-webgl:
#mochitest-webgl-e10s:
#reftest:
#reftest-e10s:
#reftest-no-accel:
#reftest-no-accel-e10s:
#web-platform-tests:
#web-platform-tests-e10s:
#web-platform-tests-reftests:
#web-platform-tests-reftests-e10s:
#xpcshell: