Skip to content

Instantly share code, notes, and snippets.

Avatar
🏠
Working from home

ibexorigin BexTuychiev

🏠
Working from home
View GitHub Profile
View 9218.py
%%time
massive_df.eval("f1001 = (f1 ** 3 + f56 ** 2 + f44 * 10) ** 0.5", inplace=True)
Wall time: 651 ms
View 9217.py
%%time
massive_df["f1001"] = crazy_function(
massive_df["f1"].values, massive_df["f56"].values, massive_df["f44"].values
)
Wall time: 201 ms
View 9216.py
import numba
@numba.jit
def crazy_function(col1, col2, col3):
return (col1 ** 3 + col2 ** 2 + col3 * 10) ** 0.5
View 9215.py
%%time
massive_df["f1001"] = crazy_function(
massive_df["f1"].values, massive_df["f56"].values, massive_df["f44"].values
)
Wall time: 324 ms
View 9214.py
massive_df = pd.concat([tps.drop(["f1000", "f1001"], axis=1)] * 10)
>>> massive_df.shape
(9579190, 120)
memory_usage = massive_df.memory_usage(deep=True)
memory_usage_in_mbs = np.sum(memory_usage / 1024 ** 2)
>>> memory_usage_in_mbs
8742.604093551636
View 9213.py
%time tps['f1001'] = crazy_function(
tps['f1'].values, tps['f56'].values, tps['f44'].values
)
Wall time: 34 ms
View 9212.py
%time tps['f1001'] = crazy_function(tps['f1'], tps['f56'], tps['f44'])
Wall time: 37 ms
View 9211.py
%time tps['f1000'] = tps.apply(lambda row: crazy_function(
row['f1'], row['f56'], row['f44']
), axis=1)
Wall time: 19.3 s
View 9210.py
def crazy_function(col1, col2, col3):
return np.sqrt(col1 ** 3 + col2 ** 2 + col3 * 10)
View 9209.py
import datatable as dt
tps = dt.fread("data/train.csv").to_pandas()
>>> tps.shape
(957919, 120)