dask example

dask_example.py

import time

import pandas as pd
import dask.dataframe as ddf


def benchmark(function, repetitions, name):
    start = time.time()
    function(repetitions)
    end = time.time()
    print('{0} seconds for {1}'.format((end - start), name))


def test_pandas(repetitions):
    df = pd.read_csv('random_people.csv')
    df['bonus'] = df['salary'] * .5
    df2 = pd.concat([df for _ in range(repetitions)])
    return df2.salary.mean()


def test_pandas_dask(repetitions):
    df = pd.read_csv('random_people.csv')
    df2 = pd.concat([df for _ in range(repetitions)])
    dd = ddf.from_pandas(df2, npartitions=3)  # why only 3 cores?
    dd['bonus'] = dd['salary'] * .5
    return dd.salary.mean().compute()


# 1,000
benchmark(test_pandas, 1000, f'{test_pandas.__name__} 1k')
benchmark(test_pandas_dask, 1000, f'{test_pandas_dask.__name__} 1k')

print('\n')

# 10,000
benchmark(test_pandas, 10000, f'{test_pandas.__name__} 10k')
benchmark(test_pandas_dask, 10000, f'{test_pandas_dask.__name__} 10k')

print('\n')

# 100,000
benchmark(test_pandas, 100000, f'{test_pandas.__name__} 100k')
benchmark(test_pandas_dask, 100000, f'{test_pandas_dask.__name__} 100k')

random_people.csv