csv_to_parquet.py

import csv
import sys
from itertools import batched

import pyarrow
from pyarrow.parquet import ParquetWriter

csv.field_size_limit(sys.maxsize)

def csv_to_parquet(csv_file, parquet_file, batch_size=10_000):
    csv_input = open(csv_file)
    reader = csv.DictReader(csv_input)

    # naively assume all columns are strings
    schema = pyarrow.schema([(name, pyarrow.string()) for name in reader.fieldnames])

    with ParquetWriter(open(parquet_file, 'wb'), schema, compression='SNAPPY') as writer:
        for rows in batched(reader, batch_size):
            table = pyarrow.Table.from_pylist(rows, schema)
            writer.write_table(table)