check-manifest.py

import csv
import os.path

with open("testinfo.data", "rb") as fp:
    reader = csv.DictReader(fp, delimiter="\t")
    for row in reader:
        path = row["id"]
        if not os.path.exists(path):
            print "%s does not exist" % path
        if row["references"]:
            groups = row["references"].split(";")
            for group in groups:
                refs = group.split(",")
                for ref in refs:
                    if ref[0] == "!":
                        ref = ref[1:]
                    if not os.path.exists(ref):
                        print "%s does not exist (ref from %s)" % (ref, path)

check-manifest2.py

from collections import defaultdict
import csv
import itertools
import json
import os.path

cats = ["testharness", "reftest", "visual", "manual", "reftest_nodes"]

old = defaultdict(set)
new = defaultdict(set)

with open("testinfo.data", "rb") as fp:
    reader = csv.DictReader(fp, delimiter="\t")
    for row in reader:
        flags = set(row["flags"].split(","))
        path = row["id"]
        if flags & {"animated", "font", "history", "interact", "paged", "speech", "userstyle"}:
            old["manual"].add(path)
        elif "script" in flags:
            old["testharness"].add(path)
        elif row["references"]:
            old["reftest"].add(path)
            old["reftest_nodes"].add(path)
            groups = row["references"].split(";")
            for group in groups:
                refs = group.split(",")
                for ref in refs:
                    if ref[0] == "!":
                        ref = ref[1:]
                    old["reftest_nodes"].add(path)
        else:
            old["visual"].add(path)

with open("../csswg-test-meta/MANIFEST.json", "rb") as fp:
    manifest = json.load(fp)
    new["testharness"] = {x["path"] for x in manifest["items"]["testharness"] if not x["path"].startswith("work-in-progress")}
    new["reftest"] = {x["path"] for x in manifest["items"]["reftest"] if not x["path"].startswith("work-in-progress")}
    new["visual"] = {x["path"] for x in manifest["items"]["visual"] if not x["path"].startswith("work-in-progress")}
    new["manual"] = {x["path"] for x in manifest["items"]["manual"] if not x["path"].startswith("work-in-progress")}
    new["reftest_nodes"] = {x["path"] for l in manifest["reftest_nodes"].values() for x in l if not x["path"].startswith("work-in-progress")}

all_old = old["testharness"] | old["reftest"] | old["visual"] | old["manual"]
all_new = new["testharness"] | new["reftest"] | new["visual"] | new["manual"]

for old_cat, new_cat in itertools.product(cats, cats):
    if old_cat != new_cat:
        if old_cat == "reftest_nodes" or new_cat == "reftest_nodes":
            continue
        moved = new[new_cat] & old[old_cat]
        if moved:
            print "tests in %s in old but %s in new:" % (old_cat, new_cat)
            print "\n".join(sorted(moved))
            print
            print
    else:
        cat = new_cat
        only_new = new[cat] - all_old
        if only_new:
            print "%s tests in new but not old" % cat
            print "\n".join(sorted(only_new))
            print
            print
        only_old = old[cat] - all_new
        if only_old:
            print "%s tests in old but not new" % cat
            print "\n".join(sorted(only_old))
            print
            print

note these both require the supersuite branch and an appropriate branch of w3ctestlib to generate the global testinfo.data