Skip to content

Instantly share code, notes, and snippets.

@dan-r95

dan-r95/memory-error.ipynb

Created August 6, 2020 14:53
Show Gist options
  • Save dan-r95/b33dacbcd81bc563184aa7660b0eb839 to your computer and use it in GitHub Desktop.
Save dan-r95/b33dacbcd81bc563184aa7660b0eb839 to your computer and use it in GitHub Desktop.
Download ZIP
memory-error.ipynb
Display the source blob
Display the rendered blob
Raw
memory-error.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "memory-error.ipynb",
"provenance": [],
"collapsed_sections": [],
"authorship_tag": "ABX9TyMIlNsdWlCkXxmFE5IyLUrm",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"accelerator": "GPU"
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/dan-r95/b33dacbcd81bc563184aa7660b0eb839/memory-error.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"metadata": {
"id": "rZhXrwK2N-8L",
"colab_type": "code",
"colab": {}
},
"source": [
"# general setup\n",
"import pandas as pd\n",
"\n",
"#track execution time\n",
"!pip install ipython-autotime\n",
"%load_ext autotime\n",
"\n",
"!pip install memory_profiler\n",
"!pip install line_profiler\n",
"%load_ext memory_profiler\n",
"%load_ext line_profiler\n",
"#for sick graphs\n",
"import plotly.express as px\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"\n",
"\n",
"!pip install filprofiler\n",
"#track gpu memory in torch\n",
"!pip install git+https://github.com/stonesjtu/pytorch_memlab\n",
"%load_ext pytorch_memlab\n",
"\n",
"\n",
"import torch\n",
"from IPython.display import clear_output\n",
"clear_output()"
],
"execution_count": 2,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "kBb9yWimOdkS",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 51
},
"outputId": "2143a1ac-a65b-4bf3-9b58-fb441d98766d"
},
"source": [
"%%writefile test.py\n",
"\n",
"sum = 0\n",
"for i in range(10000):\n",
" sum + i\n",
"print(sum)"
],
"execution_count": 12,
"outputs": [
{
"output_type": "stream",
"text": [
"Overwriting test.py\n",
"time: 2.19 ms\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "6hXo3zzKOcyQ",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 238
},
"outputId": "251c0914-8387-4722-84ba-dfcf6d83c88a"
},
"source": [
"!python -m memory_profiler cProfile test.py"
],
"execution_count": 16,
"outputs": [
{
"output_type": "stream",
"text": [
"0\n",
" 4 function calls in 0.001 seconds\n",
"\n",
" Ordered by: standard name\n",
"\n",
" ncalls tottime percall cumtime percall filename:lineno(function)\n",
" 1 0.001 0.001 0.001 0.001 test.py:2(<module>)\n",
" 1 0.000 0.000 0.001 0.001 {built-in method builtins.exec}\n",
" 1 0.000 0.000 0.000 0.000 {built-in method builtins.print}\n",
" 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n",
"\n",
"\n",
"time: 1.73 s\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "MJdzo7OZO-LU",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 51
},
"outputId": "bcb99c3d-d94c-472e-a2bf-3921931c4ca7"
},
"source": [
"!python -m memory_profiler test.py"
],
"execution_count": 14,
"outputs": [
{
"output_type": "stream",
"text": [
"0\n",
"time: 1.28 s\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "nMuftEs7QfNJ",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 391
},
"outputId": "4f3f758f-0909-458f-978d-5ac6f23a4c6e"
},
"source": [
" !git clone https://github.com/dan-r95/da-rnn-1\n",
" %cd /content/da-rnn-1/\n",
" !python3 -m cProfile memory_profiler main.py"
],
"execution_count": 17,
"outputs": [
{
"output_type": "stream",
"text": [
"Cloning into 'da-rnn-1'...\n",
"remote: Enumerating objects: 73, done.\u001b[K\n",
"remote: Counting objects: 1% (1/73)\u001b[K\rremote: Counting objects: 2% (2/73)\u001b[K\rremote: Counting objects: 4% (3/73)\u001b[K\rremote: Counting objects: 5% (4/73)\u001b[K\rremote: Counting objects: 6% (5/73)\u001b[K\rremote: Counting objects: 8% (6/73)\u001b[K\rremote: Counting objects: 9% (7/73)\u001b[K\rremote: Counting objects: 10% (8/73)\u001b[K\rremote: Counting objects: 12% (9/73)\u001b[K\rremote: Counting objects: 13% (10/73)\u001b[K\rremote: Counting objects: 15% (11/73)\u001b[K\rremote: Counting objects: 16% (12/73)\u001b[K\rremote: Counting objects: 17% (13/73)\u001b[K\rremote: Counting objects: 19% (14/73)\u001b[K\rremote: Counting objects: 20% (15/73)\u001b[K\rremote: Counting objects: 21% (16/73)\u001b[K\rremote: Counting objects: 23% (17/73)\u001b[K\rremote: Counting objects: 24% (18/73)\u001b[K\rremote: Counting objects: 26% (19/73)\u001b[K\rremote: Counting objects: 27% (20/73)\u001b[K\rremote: Counting objects: 28% (21/73)\u001b[K\rremote: Counting objects: 30% (22/73)\u001b[K\rremote: Counting objects: 31% (23/73)\u001b[K\rremote: Counting objects: 32% (24/73)\u001b[K\rremote: Counting objects: 34% (25/73)\u001b[K\rremote: Counting objects: 35% (26/73)\u001b[K\rremote: Counting objects: 36% (27/73)\u001b[K\rremote: Counting objects: 38% (28/73)\u001b[K\rremote: Counting objects: 39% (29/73)\u001b[K\rremote: Counting objects: 41% (30/73)\u001b[K\rremote: Counting objects: 42% (31/73)\u001b[K\rremote: Counting objects: 43% (32/73)\u001b[K\rremote: Counting objects: 45% (33/73)\u001b[K\rremote: Counting objects: 46% (34/73)\u001b[K\rremote: Counting objects: 47% (35/73)\u001b[K\rremote: Counting objects: 49% (36/73)\u001b[K\rremote: Counting objects: 50% (37/73)\u001b[K\rremote: Counting objects: 52% (38/73)\u001b[K\rremote: Counting objects: 53% (39/73)\u001b[K\rremote: Counting objects: 54% (40/73)\u001b[K\rremote: Counting objects: 56% (41/73)\u001b[K\rremote: Counting objects: 57% (42/73)\u001b[K\rremote: Counting objects: 58% (43/73)\u001b[K\rremote: Counting objects: 60% (44/73)\u001b[K\rremote: Counting objects: 61% (45/73)\u001b[K\rremote: Counting objects: 63% (46/73)\u001b[K\rremote: Counting objects: 64% (47/73)\u001b[K\rremote: Counting objects: 65% (48/73)\u001b[K\rremote: Counting objects: 67% (49/73)\u001b[K\rremote: Counting objects: 68% (50/73)\u001b[K\rremote: Counting objects: 69% (51/73)\u001b[K\rremote: Counting objects: 71% (52/73)\u001b[K\rremote: Counting objects: 72% (53/73)\u001b[K\rremote: Counting objects: 73% (54/73)\u001b[K\rremote: Counting objects: 75% (55/73)\u001b[K\rremote: Counting objects: 76% (56/73)\u001b[K\rremote: Counting objects: 78% (57/73)\u001b[K\rremote: Counting objects: 79% (58/73)\u001b[K\rremote: Counting objects: 80% (59/73)\u001b[K\rremote: Counting objects: 82% (60/73)\u001b[K\rremote: Counting objects: 83% (61/73)\u001b[K\rremote: Counting objects: 84% (62/73)\u001b[K\rremote: Counting objects: 86% (63/73)\u001b[K\rremote: Counting objects: 87% (64/73)\u001b[K\rremote: Counting objects: 89% (65/73)\u001b[K\rremote: Counting objects: 90% (66/73)\u001b[K\rremote: Counting objects: 91% (67/73)\u001b[K\rremote: Counting objects: 93% (68/73)\u001b[K\rremote: Counting objects: 94% (69/73)\u001b[K\rremote: Counting objects: 95% (70/73)\u001b[K\rremote: Counting objects: 97% (71/73)\u001b[K\rremote: Counting objects: 98% (72/73)\u001b[K\rremote: Counting objects: 100% (73/73)\u001b[K\rremote: Counting objects: 100% (73/73), done.\u001b[K\n",
"remote: Compressing objects: 100% (50/50), done.\u001b[K\n",
"remote: Total 176 (delta 44), reused 46 (delta 22), pack-reused 103\u001b[K\n",
"Receiving objects: 100% (176/176), 6.42 MiB | 16.04 MiB/s, done.\n",
"Resolving deltas: 100% (99/99), done.\n",
"/content/da-rnn-1\n",
"Traceback (most recent call last):\n",
" File \"/usr/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n",
" \"__main__\", mod_spec)\n",
" File \"/usr/lib/python3.6/runpy.py\", line 85, in _run_code\n",
" exec(code, run_globals)\n",
" File \"/usr/lib/python3.6/cProfile.py\", line 10, in <module>\n",
" import profile as _pyprofile\n",
" File \"/content/da-rnn-1/profile.py\", line 16, in <module>\n",
" from main_predict import preprocess_data\n",
" File \"/content/da-rnn-1/main_predict.py\", line 60, in <module>\n",
" with open(os.path.join(\"data\", \"enc_kwargs.json\"), \"r\") as fi:\n",
"FileNotFoundError: [Errno 2] No such file or directory: 'data/enc_kwargs.json'\n",
"No data collected\n",
"time: 4.6 s\n"
],
"name": "stdout"
}
]
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment