Validate that all tasks in an Evergreen config are used.

validate.py

import sys
import yaml
try:
    from yaml import CLoader as Loader
except ImportError:
    from yaml import Loader

# Load evaluated Evergreen config.
# `evaluated.yml` was created by running: `evergreen evaluate ./.evergreen/config.yml > evaluated.yml`
with open("evaluated.yml", "r") as file:
    config = yaml.load(file, Loader=Loader)

tasks = set()

# Load `tasks` section.
for task in config["tasks"]:
    task_name = task["name"]
    tasks.add (task_name)

# Load `buildvariants` section.
buildvariant_tasks = set()
for buildvariant in config["buildvariants"]:
    for task in buildvariant["tasks"]:
        if type(task) == str:
            buildvariant_tasks.add(task)
            continue
        buildvariant_tasks.add(task["name"])

# Load `task_groups` tasks.
task_groups_tasks = set()
for task_group in config["task_groups"]:
    for task in task_group["tasks"]:
        if type(task) == str:
            task_groups_tasks.add(task)
            continue
        task_groups_tasks.add(task["name"])

nonreferenced_tasks = tasks.difference (buildvariant_tasks, task_groups_tasks)
nonreferenced_tasks = list(nonreferenced_tasks)
nonreferenced_tasks.sort()
if len(nonreferenced_tasks) > 0:
    print ("Error")
    print ("`tasks` not referenced in `buildvariants` or `task_groups`:\n-{}".format("\n-".join(nonreferenced_tasks)))
    sys.exit(1)

print ("OK. All `tasks` referenced")