Python function to show progress bar while reading a CSV.

read_csv_in_chunks.py

import math
import pandas as pd

def read_csv_in_chunks(path, n_lines, **read_params):
    if 'chunksize' not in read_params or read_params['chunksize'] < 1:
        read_params['chunksize'] = 80000
    chunks = [0] * math.ceil(n_lines / read_params['chunksize'])
    for i, chunk in enumerate(pd.read_csv(path, **read_params)):
        percent = min(((i + 1) * read_params['chunksize'] / n_lines) * 100, 100.0)
        print("#" * int(percent), f"{percent:.2f}%", end='\r', flush=True)
        chunks[i] = chunk
    df = pd.concat(chunks, axis=0)
    del chunks
    print()
    return df