Crawling AWS docs for all CloudFormation resource types. Python vs. Rego!

resourcetypes.py

#!/usr/bin/env python3

import requests

def main():
    base = "https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide"
    resp = requests.get(f"{base}/toc-contents.json")

    body = resp.json()

    items = []
    for section in body["contents"]:
        if section["href"] == "template-reference.html":
            for contents in section["contents"]:
                if "contents" in contents:
                    for item in contents["contents"]:
                        if "include_contents" in item:
                            items.append(item["include_contents"])


    resources = []
    for item in items:
        r = requests.get(f"{base}/{item}")
        contents = r.json()

        for content in contents["contents"]:
            if "contents" in content:
                for item in content["contents"]:
                    if "title" in item:
                        resources.append(item["title"])

    print(resources)

if __name__ == "__main__":
    main()

resourcetypes.rego

package aws.cloudformation

base := "https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide"

all_resource_types := [resource | page := pages[_]; resource := resources_on_page(page)[_]] {
    resp := http.send({"method": "GET", "url": sprintf("%s/toc-contents.json", [base]), "force_json_decode": true})
    resp.body.contents[section].href == "template-reference.html"
    pages := [page | page := resp.body.contents[section].contents[_].contents[_].include_contents]
}

resources_on_page(doc) = [title | title := resp.body.contents[section].contents[_].title] {
    resp := http.send({"method": "GET", "url": sprintf("%s/%s", [base, doc]), "force_json_decode": true})
}