Skip to content

Instantly share code, notes, and snippets.

@dan-r95

dan-r95/memory-error.ipynb

Created August 6, 2020 15:09
Show Gist options
  • Save dan-r95/0f582d11fff06163ed34d7d9c7017b80 to your computer and use it in GitHub Desktop.
Save dan-r95/0f582d11fff06163ed34d7d9c7017b80 to your computer and use it in GitHub Desktop.
Download ZIP
memory-error.ipynb
Display the source blob
Display the rendered blob
Raw
memory-error.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "memory-error.ipynb",
"provenance": [],
"collapsed_sections": [],
"authorship_tag": "ABX9TyP4h46axeUs1UA2foM+UZq/",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"accelerator": "GPU"
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/dan-r95/0f582d11fff06163ed34d7d9c7017b80/memory-error.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"metadata": {
"id": "rZhXrwK2N-8L",
"colab_type": "code",
"colab": {}
},
"source": [
"# general setup\n",
"import pandas as pd\n",
"\n",
"#track execution time\n",
"!pip install ipython-autotime\n",
"%load_ext autotime\n",
"\n",
"!pip install memory_profiler\n",
"!pip install line_profiler\n",
"%load_ext memory_profiler\n",
"%load_ext line_profiler\n",
"#for sick graphs\n",
"import plotly.express as px\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"\n",
"\n",
"!pip install filprofiler\n",
"#track gpu memory in torch\n",
"!pip install git+https://github.com/stonesjtu/pytorch_memlab\n",
"%load_ext pytorch_memlab\n",
"\n",
"\n",
"import torch\n",
"from IPython.display import clear_output\n",
"clear_output()"
],
"execution_count": 2,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "kBb9yWimOdkS",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 51
},
"outputId": "2143a1ac-a65b-4bf3-9b58-fb441d98766d"
},
"source": [
"%%writefile test.py\n",
"\n",
"sum = 0\n",
"for i in range(10000):\n",
" sum + i\n",
"print(sum)"
],
"execution_count": 12,
"outputs": [
{
"output_type": "stream",
"text": [
"Overwriting test.py\n",
"time: 2.19 ms\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "6hXo3zzKOcyQ",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 275
},
"outputId": "2ac54a3d-ddb5-4b23-a007-dae59c9401ff"
},
"source": [
"%cd /content\n",
"!python -m memory_profiler cProfile test.py # pytorch-memlab"
],
"execution_count": 21,
"outputs": [
{
"output_type": "stream",
"text": [
"/content\n",
"0\n",
" 4 function calls in 0.001 seconds\n",
"\n",
" Ordered by: standard name\n",
"\n",
" ncalls tottime percall cumtime percall filename:lineno(function)\n",
" 1 0.001 0.001 0.001 0.001 test.py:2(<module>)\n",
" 1 0.000 0.000 0.001 0.001 {built-in method builtins.exec}\n",
" 1 0.000 0.000 0.000 0.000 {built-in method builtins.print}\n",
" 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n",
"\n",
"\n",
"time: 1.43 s\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "MJdzo7OZO-LU",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 51
},
"outputId": "bcb99c3d-d94c-472e-a2bf-3921931c4ca7"
},
"source": [
"!python -m memory_profiler test.py"
],
"execution_count": 14,
"outputs": [
{
"output_type": "stream",
"text": [
"0\n",
"time: 1.28 s\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "-gVO-qjfSl1N",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 819
},
"outputId": "9145e4b0-c54c-4fe5-be8c-a151fb2b4a13"
},
"source": [
"#!python3 main.py --task nasdaq --epochs 1"
],
"execution_count": 25,
"outputs": [
{
"output_type": "stream",
"text": [
"/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
" import pandas.util.testing as tm\n",
"2020-08-06 15:01:54,178 - VOC_TOPICS - INFO - Using computation device: cuda:0\n",
"None\n",
"2020-08-06 15:01:54,652 - VOC_TOPICS - INFO - Shape of data: (40560, 82).\n",
"Missing in data: 0.\n",
"2020-08-06 15:01:54,697 - VOC_TOPICS - INFO - Training size: 28392.\n",
"2020-08-06 15:02:04,014 - VOC_TOPICS - INFO - Iterations per epoch: 221.812 ~ 222.\n",
"2020-08-06 15:02:18,363 - VOC_TOPICS - INFO - Epoch 0, train loss: 0.017, val loss: 0.021315691918589116.\n",
"## da_rnn\n",
"\n",
"active_bytes reserved_bytes line code \n",
" all all \n",
" peak peak \n",
" 0.00B 0.00B 87 @profile \n",
" 88 def da_rnn(train_data: TrainData, n_targs: int, encoder_hidden_size=64, decoder_hidden_size=64, \n",
" 89 T=10, learning_rate=0.01, batch_size=128): \n",
" 90 \n",
" 0.00B 0.00B 91 train_cfg = TrainConfig( \n",
" 0.00B 0.00B 92 T, int(train_data.feats.shape[0] * 0.7), batch_size, nn.MSELoss()) \n",
" 0.00B 0.00B 93 logger.info(f\"Training size: {train_cfg.train_size:d}.\") \n",
" 94 \n",
" 95 enc_kwargs = { \n",
" 0.00B 0.00B 96 \"input_size\": train_data.feats.shape[1], \"hidden_size\": encoder_hidden_size, \"T\": T} \n",
" 294.00K 2.00M 97 encoder = Encoder(**enc_kwargs).to(device) \n",
" 148.50K 2.00M 98 with open(os.path.join(\"data\", \"enc_kwargs.json\"), \"w\") as fi: \n",
" 148.50K 2.00M 99 json.dump(enc_kwargs, fi, indent=4) \n",
" 100 \n",
" 148.50K 2.00M 101 dec_kwargs = {\"encoder_hidden_size\": encoder_hidden_size, \n",
" 148.50K 2.00M 102 \"decoder_hidden_size\": decoder_hidden_size, \"T\": T, \"out_feats\": n_targs} \n",
" 332.00K 2.00M 103 decoder = Decoder(**dec_kwargs).to(device) \n",
" 267.00K 2.00M 104 with open(os.path.join(\"data\", \"dec_kwargs.json\"), \"w\") as fi: \n",
" 267.00K 2.00M 105 json.dump(dec_kwargs, fi, indent=4) \n",
" 106 \n",
" 267.00K 2.00M 107 encoder_optimizer = optim.Adam( \n",
" 267.00K 2.00M 108 params=[p for p in encoder.parameters() if p.requires_grad], \n",
" 267.00K 2.00M 109 lr=learning_rate) \n",
" 267.00K 2.00M 110 decoder_optimizer = optim.Adam( \n",
" 267.00K 2.00M 111 params=[p for p in decoder.parameters() if p.requires_grad], \n",
" 267.00K 2.00M 112 lr=learning_rate) \n",
" 267.00K 2.00M 113 da_rnn_net = DaRnnNet( \n",
" 267.00K 2.00M 114 encoder, decoder, encoder_optimizer, decoder_optimizer) \n",
" 115 \n",
" 267.00K 2.00M 116 return train_cfg, da_rnn_net \n",
"No data collected\n",
"time: 41.6 s\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "nMuftEs7QfNJ",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 836
},
"outputId": "4c311296-515a-4844-9420-6ecd90315fb8"
},
"source": [
"!git clone https://github.com/dan-r95/da-rnn-1\n",
"%cd /content/da-rnn-1/\n",
"!python3 -m cProfile memory_profiler main.py"
],
"execution_count": 32,
"outputs": [
{
"output_type": "stream",
"text": [
"/content/da-rnn-1\n",
"Traceback (most recent call last):\n",
" File \"/usr/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n",
" \"__main__\", mod_spec)\n",
" File \"/usr/lib/python3.6/runpy.py\", line 85, in _run_code\n",
" exec(code, run_globals)\n",
" File \"/usr/lib/python3.6/cProfile.py\", line 10, in <module>\n",
" import profile as _pyprofile\n",
" File \"/content/da-rnn-1/profile.py\", line 16, in <module>\n",
" from main_predict import preprocess_data\n",
" File \"/content/da-rnn-1/main_predict.py\", line 80, in <module>\n",
" final_y_pred = predict(enc, dec, data, **da_rnn_kwargs)\n",
" File \"/content/da-rnn-1/main_predict.py\", line 51, in predict\n",
" _, input_encoded = encoder(numpy_to_tvar(X))\n",
" File \"/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\", line 722, in _call_impl\n",
" result = self.forward(*input, **kwargs)\n",
" File \"/content/da-rnn-1/modules.py\", line 58, in forward\n",
" weighted_input = torch.mul(attn_weights, input_data[:, t, :])\n",
"RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu!\n",
"## predict\n",
"\n",
"active_bytes reserved_bytes line code \n",
" all all \n",
" peak peak \n",
" 148.50K 2.00M 33 @profile \n",
" 34 def predict(encoder, decoder, t_dat, batch_size: int, T: int) -> np.ndarray: \n",
" 0.00B 2.00M 35 y_pred = np.zeros((t_dat.feats.shape[0] - T + 1, t_dat.targs.shape[1])) \n",
" 36 \n",
" 0.00B 2.00M 37 for y_i in range(0, len(y_pred), batch_size): \n",
" 0.00B 2.00M 38 y_slc = slice(y_i, y_i + batch_size) \n",
" 0.00B 2.00M 39 batch_idx = range(len(y_pred))[y_slc] \n",
" 0.00B 2.00M 40 b_len = len(batch_idx) \n",
" 0.00B 2.00M 41 X = np.zeros((b_len, T - 1, t_dat.feats.shape[1])) \n",
" 0.00B 2.00M 42 y_history = np.zeros((b_len, T - 1, t_dat.targs.shape[1])) \n",
" 43 \n",
" 0.00B 2.00M 44 for b_i, b_idx in enumerate(batch_idx): \n",
" 0.00B 2.00M 45 idx = range(b_idx, b_idx + T - 1) \n",
" 46 \n",
" 0.00B 2.00M 47 X[b_i, :, :] = t_dat.feats[idx, :] \n",
" 0.00B 2.00M 48 y_history[b_i, :] = t_dat.targs[idx] \n",
" 49 \n",
" 4.50K 2.00M 50 y_history = numpy_to_tvar(y_history) \n",
" 11.54M 22.00M 51 _, input_encoded = encoder(numpy_to_tvar(X)) \n",
" 52 y_pred[y_slc] = decoder(input_encoded, y_history).cuda().data.numpy() \n",
" 53 \n",
" 54 return y_pred \n",
"time: 7.49 s\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "n9exredTTz2a",
"colab_type": "code",
"colab": {}
},
"source": [
""
],
"execution_count": null,
"outputs": []
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment