csv_to_parquet.py

csv_to_parquet.py

"""CLI tool to stream CSV (from stdin) to parquet"""
import sys
from pathlib import Path
from typing import TextIO

import pandas as pd

CHUNKSIZE=2**16 # ~65k rows per parquet chunk

def convert(instream: TextIO, outdir: Path, chunksize=CHUNKSIZE):
    outdir.mkdir()
    for idx, chunk in enumerate(pd.read_csv(instream, chunksize=chunksize)):
        chunk.to_parquet(outdir / f"{idx:010}.parquet")

if __name__ == "__main__":
    convert(sys.stdin, Path(sys.argv[1]))