Convert CSV to NDJSON (LDJSON)

csv_to_ndjson.py

#!/usr/bin/env python3
#
# csv_to_ndjson.py
#
# Author: Werner Robitza
#
# Convert CSV files to .ndjson.
# This assumes the same keys being used in every line of the input file.
# It works on a line-by-line basis, so it should be fast and memory-efficient.

import json
import argparse
import csv
from tqdm import tqdm


def convert_to_ndjson(csv_file, ndjson_file):
    with open(csv_file, "r") as in_f:
        with open(ndjson_file, "w") as out_f:
            csv_reader = csv.DictReader(in_f)
            for row in tqdm(csv_reader):
                out_f.write(json.dumps(row) + "\n")


def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter
    )
    parser.add_argument("input", help="input .csv")
    parser.add_argument("output", help="output .ndjson")
    cli_args = parser.parse_args()

    convert_to_ndjson(cli_args.input, cli_args.output)


if __name__ == "__main__":
    main()