Test all webappanalyzer patterns in Python

test_compile.py

import glob
import re

import orjson as json
from tqdm import tqdm

good_keys: list[str] = [
    "cookies",
    "dom",
    "dns",
    "js",
    "headers",
    "text",
    "css",
    "probe",
    "robots",
    "url",
    "xhr",
    "meta",
    "scriptSrc",
    "scripts",
    "html",
]


confidence = re.compile("\\\\;confidence:[0-9]+")


def _clean_pattern(pattern: str) -> str:
    pattern = re.sub(confidence, "", pattern)
    pattern = pattern.replace("\\;version:\\1", "").replace("\\;version:\\2", "")
    return pattern


def _compile_str(string: str):
    re.compile(_clean_pattern(string))


def _parse(data: str | list | dict) -> None:
    if isinstance(data, str) is True:
        _compile_str(data)
    elif isinstance(data, list) is True:
        for item in data:
            _parse(data[item])
    elif isinstance(data, dict):
        for k in data.keys():
            _parse(data[k])
    else:
        pass


def test_compile_all_regex():
    errors: list[str] = []
    data: dict = {}
    for file in tqdm(glob.glob("technologies/*.json")):
        with open(file, "r") as fp:
            tmp = json.loads(fp.read())
            data = data | tmp

    for key in data:
        for elt in data[key]:
            if elt in good_keys:
                try:
                    _compile_str(data[key][elt])
                except (TypeError, re.error):
                    errors.append(f"{key} - {elt}")

    print(errors)