Issue with LR Finder

lr_find_issue.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/muellerzr/9867b45df67e4c31a1db415c409f1dea/scratchpad.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "lIYdn1woOS1n"
      },
      "outputs": [],
      "source": [
        "!pip install -U fastai >> /dev/null"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from fastai.vision.all import *"
      ],
      "metadata": {
        "id": "lmp-vMxg5pU9"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "set_seed(99, True)\n",
        "path = untar_data(URLs.PETS)/'images'\n",
        "dls = ImageDataLoaders.from_name_func(\n",
        "    path, get_image_files(path), valid_pct=0.2,\n",
        "    label_func=lambda x: x[0].isupper(), item_tfms=Resize(224))"
      ],
      "metadata": {
        "id": "NTBscbjK58Bi"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "First we train for one epoch, which reduces our loss greatly"
      ],
      "metadata": {
        "id": "L-qh-PVlKlbk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "learn = vision_learner(dls, resnet34, metrics=error_rate)\n",
        "learn.fine_tune(1)"
      ],
      "metadata": {
        "id": "_aOqHwNR6BxN",
        "outputId": "8c64a21a-6afd-4a08-ed37-3e1926f76c61",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 144
        }
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "<style>\n",
              "    /* Turns off some styling */\n",
              "    progress {\n",
              "        /* gets rid of default border in Firefox and Opera. */\n",
              "        border: none;\n",
              "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "        background-size: auto;\n",
              "    }\n",
              "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "        background: #F44336;\n",
              "    }\n",
              "</style>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>error_rate</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.147566</td>\n",
              "      <td>0.031143</td>\n",
              "      <td>0.008119</td>\n",
              "      <td>00:58</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "<style>\n",
              "    /* Turns off some styling */\n",
              "    progress {\n",
              "        /* gets rid of default border in Firefox and Opera. */\n",
              "        border: none;\n",
              "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "        background-size: auto;\n",
              "    }\n",
              "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "        background: #F44336;\n",
              "    }\n",
              "</style>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>error_rate</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.045159</td>\n",
              "      <td>0.011910</td>\n",
              "      <td>0.003383</td>\n",
              "      <td>01:09</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Then we modify `after_batch` to print some useful bits to look at. \n",
        "\n",
        "Specifically I'm interested in values that cause any `CancelFitException`'s, which would be our `smooth_loss`, `best_loss`, `train_iter`, and `num_it`"
      ],
      "metadata": {
        "id": "KBnKS7omKm_t"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "learn.save(\"my_start_point\")"
      ],
      "metadata": {
        "id": "LDLatnUWNU3K"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "@patch\n",
        "def after_batch(self:LRFinder):\n",
        "    super(LRFinder, self).after_batch()\n",
        "    if self.smooth_loss < self.best_loss: \n",
        "      print(\"Smooth loss lower than best loss, new best loss\")\n",
        "      self.best_loss = self.smooth_loss\n",
        "    if self.smooth_loss > 4*self.best_loss and self.stop_div: \n",
        "      print(\"Smooth loss was greater than 4* the best loss, stop_div is `True`\")\n",
        "      print(f'`smooth_loss`: {self.smooth_loss}')\n",
        "      print(f'4x `best_loss`: {4*self.best_loss}')\n",
        "      raise CancelFitException()\n",
        "    if self.train_iter >= self.num_it: \n",
        "      print(\"Number of training iterations > num_it\")\n",
        "      raise CancelFitException()"
      ],
      "metadata": {
        "id": "Ah5uYRkE6K6Z"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We try lr find again:"
      ],
      "metadata": {
        "id": "sfahiWMDNb9G"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "learn.lr_find()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 422
        },
        "id": "GRBDY-iPMOCc",
        "outputId": "632ab8dc-b4d9-44f3-de11-025a2e8da43f"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "<style>\n",
              "    /* Turns off some styling */\n",
              "    progress {\n",
              "        /* gets rid of default border in Firefox and Opera. */\n",
              "        border: none;\n",
              "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "        background-size: auto;\n",
              "    }\n",
              "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "        background: #F44336;\n",
              "    }\n",
              "</style>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              ""
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Smooth loss lower than best loss, new best loss\n",
            "Smooth loss lower than best loss, new best loss\n",
            "Smooth loss lower than best loss, new best loss\n",
            "Smooth loss was greater than 4* the best loss, stop_div is `True`\n",
            "`smooth_loss`: 0.03686267137527466\n",
            "4x `best_loss`: 0.035835009068250656\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "IndexError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-6-d81c6bd29d71>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlr_find\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/fastai/callback/schedule.py\u001b[0m in \u001b[0;36mlr_find\u001b[0;34m(self, start_lr, end_lr, num_it, stop_div, show_plot, suggest_funcs)\u001b[0m\n\u001b[1;32m    298\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mfunc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtuplify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msuggest_funcs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    299\u001b[0m             \u001b[0mnms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpartial\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# deal with partials\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 300\u001b[0;31m             \u001b[0m_suggestions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlosses\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_it\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    301\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    302\u001b[0m         \u001b[0mSuggestedLRs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnamedtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'SuggestedLRs'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/fastai/callback/schedule.py\u001b[0m in \u001b[0;36mvalley\u001b[0;34m(lrs, losses, num_it)\u001b[0m\n\u001b[1;32m    225\u001b[0m     \u001b[0midx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax_start\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msections\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msections\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    226\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 227\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlrs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlrs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlosses\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    228\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    229\u001b[0m \u001b[0;31m# Cell\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mIndexError\u001b[0m: index 0 is out of bounds for dimension 0 with size 0"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We see the trigger point, smooth loss was greater than 4x the loss, and stop_div was True"
      ],
      "metadata": {
        "id": "ItSYc2QmNeET"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now we investigate what our suggestion methods use to look at bits, which is looking at `learn.recorder.lrs` and `learn.recorder.losses`. `num_iters` is 100, hence the 100:"
      ],
      "metadata": {
        "id": "TfX8NKQfNhpQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "lrs, losses = tensor(learn.recorder.lrs[100//10:-5]), tensor(learn.recorder.losses[100//10:-5])"
      ],
      "metadata": {
        "id": "7t9fj97L7RES"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "learn.recorder.lrs"
      ],
      "metadata": {
        "id": "83LimerH7W93",
        "outputId": "08b68002-7e9c-45f4-ac19-0e7f5bd3f752",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1e-07,\n",
              " 1.202264434617413e-07,\n",
              " 1.4454397707459274e-07,\n",
              " 1.7378008287493754e-07,\n",
              " 2.0892961308540395e-07,\n",
              " 2.51188643150958e-07,\n",
              " 3.019951720402016e-07,\n",
              " 3.6307805477010137e-07]"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "lrs, losses"
      ],
      "metadata": {
        "id": "Gg7WLSg_7VmY",
        "outputId": "4f569159-496a-45b4-91a3-f718bd7b260d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([]), tensor([]))"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We get empty tensors because we stopped *so* early that our `100//10:-5` wasn't actually getting us anything!"
      ],
      "metadata": {
        "id": "WOYixzHzNpPk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "learn.recorder.lrs[100//10:-5]"
      ],
      "metadata": {
        "id": "pzmr-8qM6ay9",
        "outputId": "1ee121b6-5ab6-4116-f6e8-855a379f4646",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "scratchpad.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}