Consume a common crawn (WARC) file to parquet table

to-parquet.py

import gzip
import os
import pathlib

import warcio.archiveiterator
from bs4 import BeautifulSoup
import pandas


def warc_stream(stream):
    uri: str | None = None
    for record in warcio.archiveiterator.ArchiveIterator(stream):
        uri = record.rec_headers.get("WARC-Target-URI")
        cs = record.content_stream().read()

        try:
            cs = gzip.decompress(cs)
        except:
            pass

        soup = BeautifulSoup(cs, "html.parser")
        try:
            if uri:
                yield {"url": uri, "text": (soup.getText() or "")}
        except Exception as e:
            print(f"    URI {uri} was not an HTML stream: {e}")
        uri = None


files = pathlib.Path(".").glob("*.warc.gz")
for filename in files:
    print(filename)
    with open(filename, "rb") as stream:
        df = pandas.DataFrame(data=warc_stream(stream))
        df.to_parquet(os.path.basename(filename) + ".parquet")