Convert hlist to parquet format

hlist2parquet.py

#/usr/bin/env python

import sys
import pyarrow as pa
import pyarrow.parquet as pq
from helpers.io_utils import hlist2pandas
import tqdm

def main(input_filename):
    reader = hlist2pandas(input_filename, sanitize_column_names=True, chunksize=1000000)
    table = pa.Table.from_pandas(next(reader), preserve_index=False)
    schema = table.schema
    with pq.ParquetWriter(input_filename.replace(".list", ".parquet"), schema, compression="snappy") as writer:
        writer.write_table(table)
        for table in tqdm.tqdm(reader):
            writer.write_table(pa.Table.from_pandas(table, preserve_index=False))

if __name__ == "__main__":
    main(sys.argv[1])