data_formats.py

import pandas as pd
import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import fastavro
import os

np.random.seed(0)  # for reproducibility

region = np.random.choice(["Europe", "North America", "Latin America", "Asia"], 10000000, p=[0.3, 0.5, 0.1, 0.1])
sector = np.random.choice(["Technology", "Healthcare", "Finance", "Consumer"], 10000000, p=[0.3, 0.5, 0.1, 0.1])
product = np.random.choice(["AC" + str(i) for i in range(1, 100001)], 10000000)
spend = np.random.uniform(10, 100000, 10000000)
demand = np.random.binomial(1, 0.8, 10000000)
date = np.random.choice(pd.date_range(start='2018-01-01', end='2022-11-01'), 10000000)

df = pd.DataFrame({
    "Region": region,
    "Sector": sector,
    "Product": product,
    "Spend": spend,
    "Demand": demand,
    "Date": date
})

df.head()

df.to_csv('data.csv', index=False)
df.to_feather('data.arrow')
df.to_parquet('data.parquet')
df.to_parquet('data2.parquet', compression = 'gzip')

csv_size = os.path.getsize('data.csv')
arrow_size = os.path.getsize('data.arrow')
parquet_size = os.path.getsize('data.parquet')
parquet2_size = os.path.getsize('data2.parquet')


print(csv_size, arrow_size, parquet_size, parquet2_size)