A simple generic importer for DuckDB when you try to import a big CSV datafile

duckdbGenericImporter.py

import duckdb, csv
// opening (or creating) a duckdb file. If you want, put ':memory:' for in-memory database
con=duckdb.connect(<dbfile>) 
con.sql('PRAGMA memory_limit="6GB"')    // helps by limiting RAM usage from duckdb
con.sql('PRAGMA temp_directory="tmp/"') // helps by setting a tempfiles directory
a=0
with open(<csvfile>) as csvfile:
    data = csv.reader(csvfile)
    for i in data:
        # print(','.join(i))
        sql="insert into <table> values ({},{},{},{}) on conflict do nothing".format(i[0],i[1],i[2].strip(),i[3])
        con.execute(sql)
        a+=1
        print(a) // show number of lines inserted