List requests present in WARC files

list_warc_content.py

import glob
from warcio import ArchiveIterator

for warc in glob.glob("output/.tmph919m5n3/collections/crawl-*/archive/*.warc.gz"):
    with open(warc, "rb") as fh:
        for record in ArchiveIterator(fh):
            if record.rec_type == "request":
                print(record.rec_headers.get_header('WARC-Target-URI'))

print("DONE")