Split CSV into files of up to N rows

split_csv.py

import glob
import pandas as pd
import math
import os

max_rows = 100000

for file in glob.glob('*.csv'):
    filename = os.path.splitext(file)[0]
    print(f'Loading {file}...')
    df = pd.read_csv(file, sep=',', low_memory=False)
    
    if len(df) > max_rows:
        print(f'Processing into chunks of {max_rows} rows...')
        for i in range(math.ceil(len(df) / max_rows)):
            split = df.head(max_rows)
            split.to_csv(f'{filename}_{i+1}.csv')
            print(f'Wrote {filename}_{i+1}.csv...')
            df = df.iloc[max_rows:]
    else:
        print('Skipping...')
        
print('Done!')