Loading and Training data in chunk

chunk_train.py

>>> from sklearn.linear_model import SGDRegressor
>>> from sklearn.datasets import make_regression
>>> import numpy as np
>>> import pandas as pd

>>> ### Load original data
>>> original_data = pd.read_csv('sample.csv')
>>> print(f'Shape of original data {original_data.shape:.f02}')
Shape of original data (100000, 21)


>>> ### Load in chunk
>>> chunksize = 1000
>>> reg = SGDRegressor()
>>> features_columns = [str(i) for i in range(20)]

>>> ### Fit each chunk
>>> for train_df in pd.read_csv("sample.csv", chunksize=chunksize, iterator=True):
>>>     X = train_df[features_columns]
>>>     Y = train_df["target"]
>>>     reg.partial_fit(X, Y)

### The reg.partial_fit() method fit each chunk at a time and update weights accordingly after each the next chunk is loaded