CSV to JSONL script using pandas & click

csv2jsonl

#!/usr/bin/env python

import csv
import json
import sys

csv.field_size_limit(sys.maxsize)

import click
import pandas as pd

stdin = click.get_binary_stream('stdin')
stdout = click.get_text_stream('stdout')

@click.command()
@click.option('--chunksize', default=100)
@click.argument('csv_file', type=click.File('rbU'))
def csv2jsonl(csv_file, chunksize):
    for chunk in pd.read_csv(csv_file, chunksize=chunksize):
        chunk.to_json(stdout, orient='records', lines=True)

if __name__=='__main__':
    csv2jsonl()

Drop it somewhere on your path and you can run either

csv2jsonl my_csv.csv

or

csv2jsonl - < my_csv.csv