compress_csv.py

from pathlib import Path
import pandas as pd

DATA_FOLDER = Path("data")

# get all csv files in data and its subfolders
for csv_file in DATA_FOLDER.rglob("*.csv"):

	csv_full_path = csv_file.resolve()

	# read csv file to dataf and compress
	dataf = pd.read_csv(csv_full_path)
	dataf.to_csv(f"{csv_full_path}.bz2", 
				 compression='bz2',
				 index=False)

	# delete the uncompressed csv file
	csv_file.unlick()
	

# Note ##########################################
#  read with pandas:							#
#	dataf = pd.read_csv(						#
#			'file.csv.gzip', compression='bz2')	#	
#  compression:									#
#	'bz2' > 'xz' > 'gzip'						#
#    when dealing with floats type data 		#
#    only										#
#	'xz' > 'bz2'> 'gzip' 						#
#	when dealing with  mixed data types.		#
#	 'xz' takes longer writing and reading.		#
#												#
#################################################