A simple example demonstrating how to use the Python memory_profiler package.

run.py

#!/usr/bin/env python

import pandas as pd
from sklearn.ensemble import RandomForestClassifier


@profile
def infinite_loading():
    """Exceeds available memory"""
    train = pd.read_csv('big.csv')
    while True:
        new = pd.read_csv('big.csv')
        train = pd.concat([train, new])


@profile
def python_loading():
    """Loads via Python builtins"""
    X = pd.DataFrame([line.split(',')[0:2] for line in open('train.csv') if not line.startswith('id')])
    y = pd.DataFrame([line.split(',')[2] for line in open('train.csv') if not line.startswith('id')])
    estimator = RandomForestClassifier(n_estimators=100)
    estimator.fit(X, y)


@profile
def sane_pandas_load():
    """Compares to redundant_pandas_load"""
    train = pd.read_csv('train.csv')
    X = train.drop('target', axis=1).values
    y = train.target.values
    estimator = RandomForestClassifier(n_estimators=100)
    estimator.fit(X, y)


@profile
def redundant_pandas_load():
    """Runs a classifier"""
    X = pd.read_csv('train.csv').drop('target', axis=1).values
    y = pd.read_csv('train.csv').target.values
    estimator = RandomForestClassifier(n_estimators=100)
    estimator.fit(X, y)


if __name__ == '__main__':
    redundant_pandas_load()
    sane_pandas_load()
    infinite_loading()
    python_loading()

train.csv