LR Scheduler which works with TF 2.x distributed mode

lr_scheduler_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensorflow version 2.0.0\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.framework import constant_op\n",
    "from tensorflow.python.framework import ops\n",
    "from tensorflow.python.ops import control_flow_ops\n",
    "from tensorflow.python.ops import math_ops\n",
    "from tensorflow.python.keras.optimizer_v2.learning_rate_schedule import LearningRateSchedule\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "print(\"Tensorflow version \" + tf.__version__)\n",
    "\n",
    "class CosineDecayWithWarmup(LearningRateSchedule):\n",
    "    \"\"\"A LearningRateSchedule that uses a cosine decay schedule.\"\"\"\n",
    "\n",
    "    def __init__(\n",
    "            self,\n",
    "            initial_learning_rate,\n",
    "            max_learning_rate,\n",
    "            warmup_steps,\n",
    "            decay_steps,\n",
    "            alpha=0.0,\n",
    "            name=None):\n",
    "        super().__init__()\n",
    "        self.initial_learning_rate = initial_learning_rate\n",
    "        self.max_learning_rate = max_learning_rate\n",
    "        self.warmup_steps = warmup_steps\n",
    "        self.decay_steps = decay_steps\n",
    "        self.alpha = alpha\n",
    "        self.name = name\n",
    "\n",
    "    @staticmethod\n",
    "    def lr_warmup(steps, warmup_steps, max_learning_rate, initial_learning_rate):\n",
    "        return initial_learning_rate + (\n",
    "            max_learning_rate - initial_learning_rate\n",
    "        ) * (steps / warmup_steps)\n",
    "\n",
    "    @staticmethod\n",
    "    def cosine_decay(steps, warmup_steps, decay_steps, max_learning_rate, alpha):\n",
    "        completed_fraction = (\n",
    "            steps - warmup_steps) / decay_steps\n",
    "        cosine_decayed = 0.5 * (1.0 + math_ops.cos(\n",
    "            constant_op.constant(math.pi) * completed_fraction))\n",
    "        decayed = (1 - alpha) * cosine_decayed + alpha\n",
    "        return math_ops.multiply(max_learning_rate, decayed)\n",
    "\n",
    "    def __call__(self, step):\n",
    "        with ops.name_scope_v2(self.name or \"CosineDecayWithWarmup\"):\n",
    "            initial_learning_rate = ops.convert_to_tensor(\n",
    "                self.initial_learning_rate, name=\"initial_learning_rate\")\n",
    "            max_learning_rate = ops.convert_to_tensor(\n",
    "                self.max_learning_rate, name=\"initial_learning_rate\")\n",
    "            dtype = initial_learning_rate.dtype\n",
    "            decay_steps = math_ops.cast(self.decay_steps, dtype)\n",
    "            warmup_steps = math_ops.cast(self.warmup_steps, dtype)\n",
    "            total_steps = decay_steps + warmup_steps\n",
    "\n",
    "            global_step_recomp = math_ops.cast(step, dtype)\n",
    "            global_step_recomp = math_ops.minimum(\n",
    "                global_step_recomp, total_steps)\n",
    "\n",
    "            return control_flow_ops.cond(\n",
    "                math_ops.less_equal(global_step_recomp, warmup_steps),\n",
    "                lambda: self.lr_warmup(\n",
    "                    global_step_recomp, warmup_steps, max_learning_rate,\n",
    "                    initial_learning_rate\n",
    "                ),\n",
    "                lambda: self.cosine_decay(\n",
    "                    global_step_recomp, warmup_steps, decay_steps,\n",
    "                    max_learning_rate, self.alpha\n",
    "                )\n",
    "            )\n",
    "\n",
    "    def get_config(self):\n",
    "        return {\n",
    "            \"initial_learning_rate\": self.initial_learning_rate,\n",
    "            \"decay_steps\": self.decay_steps,\n",
    "            \"alpha\": self.alpha,\n",
    "            \"name\": self.name\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "lr_schedule = CosineDecayWithWarmup(\n",
    "    initial_learning_rate=1e-5, max_learning_rate=5e-4,\n",
    "    warmup_steps=200,\n",
    "    decay_steps=800,\n",
    "    alpha=1e-6\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate schedule:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXhV1bn48e+bk3mEjAQChCECARnDpNQJGWqt2EqvoCJVLLWC1tpJf7e1rfd6r7a9WrmKihVRVIZrW0XrNRHECWUIMp5AIIBAgJMJCAlkzvr9cbbeEAM5gST7DO/nefJwzt5rr/MuNpw3e6+11xJjDEoppdRXguwOQCmllHfRxKCUUuosmhiUUkqdRRODUkqps2hiUEopdZZguwNoD4mJiSY9Pd3uMJRSyqds3ry51BiT1Hy7XySG9PR0cnNz7Q5DKaV8iogcbGm73kpSSil1Fk0MSimlzqKJQSml1Fk0MSillDqLJgallFJn8SgxiMhUEckXkQIRebCF/WEissLav0FE0pvse8jani8iU1qrU0SWiMgBEdlq/Qy/uCYqpZRqi1aHq4qIA3gGmAQUAptEZJUxJq9JsTnACWNMfxGZATwO3CwimcAMYDDQHVgtIpdYx5yvzl8aY95oh/YppZRqI0+eYxgDFBhj9gOIyHJgGtA0MUwDfm+9fgN4WkTE2r7cGFMDHBCRAqs+PKhTdYDP95Wx4UAZYcEOwoKDCA9x0DUyhMSYMBKjw0iOCSMqzC8eb1FKXSBPvgF6AIebvC8Exp6rjDGmXkTKgQRr+/pmx/awXp+vzkdF5GFgDfCglVjOIiJzgbkAvXr18qAZyhjDAyu3cqy8+rzlkmPC6JsURZ/EaAZ2i2FoWhyDUmMJD3F0UqRKKTt546+GDwEuIBRYBPwaeKR5IWPMIms/WVlZutqQB7YXlnOsvJo/Th/Kd4d2p6a+geq6Ro6frqW0sobSyhqOlVdzoPQ0B0pP897OYyzbeAiAEIcwKDWWcX0TuLx/ImPS44kI1UShlD/yJDEcAXo2eZ9mbWupTKGIBANxQFkrx7a43RhzzNpWIyIvAb/wIEblgWynC0eQMDkzhYhQx9df7N3iwlssb4zhWHk12wtPsq2wnC8OnmDJui9Z9PF+Qh1BjO7TlalDUpkyOIXkmJbrUEr5Hk8SwyYgQ0T64P7yngHc0qzMKmA28DkwHfjAGGNEZBXwuog8gbvzOQPYCMi56hSRVGPMMauP4kZg50W2UVly8ooY2yeeLpGhHpUXEbp3iaB7lwimDkkF4ExtPRsPHOfTvaV8sLuY3765k4ff2smY9HiuH5rKDcN6EBcZ0pHNUEp1sFYTg9VnMB/IBhzAYmOMU0QeAXKNMauAF4GlVufycdxf9FjlVuLuVK4H5hljGgBaqtP6yNdEJAl38tgK3N1+zQ1c+0oqKSiu5LaxF9cfExkazFUDkrlqQDL/+p1B7Cmq5N0dx3h3xzF++5aTf//nLq67NJWbR/dkbJ943PldKeVLxBjfvz2flZVldHbV83v2w308/t5uPnvwGrp3ieiQz9h5pJzlmw7x1pajVNTU0zcpirsm9OX7I3tox7VSXkhENhtjsr6xXRNDYLjxmXU0NBrevndCh39WVW0D/9xxjCWfHWDnkVMkRIVy+/h0Zo3vTXyUZ7exlFId71yJQafECABFp6rZevgkUwandMrnRYQ6mD4qjbfnT2DZj8YxrGcXnly9hwmPf8Dj7+3m5JnaTolDKXVhvHG4qmpnOXlFAEwe3K1TP1dEGN8vgfH9EthbVMF/f1DAcx/tY+nnB5kzoQ9zvtWH2HDtqFbK2+gVQwDIcbrokxhFRnK0bTFkpMSwYOYI3vvpFUzon8hTa/Zy5R/XsnT9QeobGm2LSyn1TZoY/Fx5VR2f7ytjcmaKV4wQGtAthudmjeKdeycwsFssv31zJ99Z8Cmf7i21OzSllEUTg5/7ML+Y+kbT6beRWjOkRxyv/2gsz902iqq6Bm57cQM/eiWXIyer7A5NqYCnicHPZTtdJMWEMaJnF7tD+QYRYeqQbuT87Ap+PXUg6wpKmfTERyz+9AANjb4/Wk4pX6WJwY9V1zXwYX4JkzJTCAqy/zbSuYSHOPjJVf3I+dkVjO0TzyPv5PH9hevIO3rK7tCUCkiaGPzYuoJSztQ2MDmzc4apXqy0rpEs/uFoFswcwZGTVXz36U954v091GnntFKdShODH8txFhETFsxl/RLtDsVjIsINw7qz+oEruWFYdxas2ctNz37GvpJKu0NTKmBoYvBTDY2G1buKuGpgMqHBvneau0SG8uTNw1l460gOHT/DdxZ8wiuff4k/PKmvlLfzvW8M5ZHNB09Qdrq205527ijXXZpKzv1XMLZPAg+/5eTOJZs4cVqfnFaqI2li8FPZThehjiCuvCTJ7lAuWnJsOEvuGM0j0wazrqCM6xZ8wuaDx+0OSym/pYnBDxljyMlzcXn/BGL8ZMoJEeH28en87SeXEeII4ubn1/PCx/v11pJSHUATgx/adayCw8ervO6htvZwaVocb987gYmDknn03V386JXNlFfV2R2WUn5FE4Mfyna6EIFrB/l2/8K5xEWE8Nxto3j4+kw+zC/mewvX6aglpdqRJgY/lJNXRFbvriTFhNkdSocREe6c0IfX7hpL+Zk6bnx6HWt3F9sdllJ+QRODnzl8/Ay7jp1icqb/3UZqydi+Cbw1/3J6xkdy58ubeO6jfdrvoNRF0sTgZ7KdLgAm+/gw1bZI6xrJGz8Zz3WXpvLY/+7mZyu2UlPfYHdYSvksXajHz+Q4ixjYLYbeCVF2h9KpIkODeXrmCAZ1i+HPOXs4Wl7Nolmj6BKpS4kq1VZ6xeBHSitryD143C9HI3lCRJh/TQZPzRjO1kMnuenZzzh8/IzdYSnlczQx+JE1u4poNPjMpHkdZdrwHiydM4aSihq+t3Ad2w6ftDskpXyKJgY/kuMsokeXCAZ3j7U7FNuN7ZvA3++5jPAQBzMWrWfNriK7Q1LKZ2hi8BOVNfV8UlDK5MHesYSnN+ifHMM/7rmcjJRo5i7dzD+2FNodklI+QRODn/h4Twm19Y1MCdD+hXNJignj9R+NY2yfeH62YhtL1h2wOySlvJ4mBj+R7XTRNTKErN5d7Q7F60SHBbP4h6OZnJnC79/O46nVe/VZB6XOQxODH6itb+SD3cVcOyiFYIee0paEhzhYeOtIpo9K48nVe3jknTwadV1ppVqkzzH4gfX7y6iorg/YYaqeCnYE8cebhhIbHsLidQc4VVXPH6cPxeHF62ErZQdNDH4gJ89FRIiDb2X4zhKedgkKEn57/SDiIkJ4cvUeGhob+fMPhumVllJNaGLwcY2NhhxnEVdekkR4iMPucHyCiPDTazMIdgh/ys6nwcCT/6LJQamvaGLwcdsKT1JcUcOUIYH9UNuFmHd1fxxBwmP/u5vGRsNfZgwnRJODUp51PovIVBHJF5ECEXmwhf1hIrLC2r9BRNKb7HvI2p4vIlPaUOcCEdFJ9luR7SwiOEi4ZoAmhgtx95X9+NfrBvHPHce4b9kW6hoa7Q5JKdu1mhhExAE8A3wbyARmikhms2JzgBPGmP7Ak8Dj1rGZwAxgMDAVWCgijtbqFJEsQMddeiAnz8W4vgnERfrHEp52+NEVffnt9Zn8704X8177gtp6TQ4qsHlyxTAGKDDG7DfG1ALLgWnNykwDXrZevwFMFPfjt9OA5caYGmPMAaDAqu+cdVpJ40/Ary6uaf6voLiC/SWnA2qK7Y4yZ0Iffv/dTHLyirh/xRbq9cpBBTBPEkMP4HCT94XWthbLGGPqgXIg4TzHnq/O+cAqY8yx8wUlInNFJFdEcktKSjxohv/Jdrrn/5kU4JPmtZcfXt6H33xnEO/ucPGrN7brcw4qYHlV57OIdAd+AFzVWlljzCJgEUBWVlZA/g/OcboYlhZHalyE3aH4jbu+1Zfqugb+nLOHsBAH//G9ITr3lAo4niSGI0DPJu/TrG0tlSkUkWAgDihr5diWto8A+gMF1n/GSBEpsPouVBPHyqvYVljOL6cMsDsUvzP/mgzO1Daw8MN9RIQ4+O31gzQ5qIDiSWLYBGSISB/cX94zgFualVkFzAY+B6YDHxhjjIisAl4XkSeA7kAGsBGQluo0xjiBrx/fFZFKTQotez/PfRtJJ83rGL+cMoCqugYWrztAZKiDX2gCVgGk1cRgjKkXkflANuAAFhtjnCLyCJBrjFkFvAgsFZEC4DjuL3qsciuBPKAemGeMaQBoqc72b57/yna66JsURf/kaLtD8UsiwsPXZ1Jd18DTawuICHUw72r9HUUFBo/6GIwx7wLvNtv2cJPX1bj7Blo69lHgUU/qbKGMfuu1oPxMHev3H2fuFX3tDsWviQj/fuOlVNc18qfsfGIjQpg1rrfdYSnV4byq81l5Zs3uIhoaTcAv4dkZHEHCn6YP5VRVHQ+/tZP4yFC+MzTV7rCU6lD6/L8PynEWkRIbxrC0LnaHEhCCHUE8fctIRvXqys9WbGVdQandISnVoTQx+JjqugY+2lPCpMwUgnS66E4TEergxdmj6ZMYxdxXctlRWG53SEp1GE0MPuaTvaVU1TXoaCQbxEWG8MqcMXSJDOWHL23kQOlpu0NSqkNoYvAx2U4XMeHBjO2TYHcoASklNpylc8ZggFkvbqDoVLXdISnV7jQx+JD6hkbW7Cpi4sBkQoP11Nmlb1I0S+4YzYnTtcxevJHyqjq7Q1KqXem3iw/Z9OUJTpyp0yU8vcDQtC48PyuLfSWV/OTVzTojq/Irmhh8SE6ei9DgIK68JMnuUBQwISORx74/lM/2lfHg37ZjTEBO2aX8kD7H4COMcS/h+a3+iUSF6WnzFjeNSuPIySqeeH8PafGRPDDpErtDUuqi6RWDj3AePcWRk1U6GskL3XtNf/4lK40Fa/ayctPh1g9Qysvpr54+IsfpIkhg4qBku0NRzYgIj37vUo6VV/P//rGDbnHhXKG3+5QP0ysGH5GTV0RWejwJ0WF2h6JaEOIIYuGtI+mfHM09r31B3tFTdoek1AXTxOADDpadZrerQudG8nIx4SG8dMdoosOCuXPJJo6VV9kdklIXRBODD8hx6toLviI1LoLFPxxNZU09d7y0icqaertDUqrNNDH4gGyni0GpsfSMj7Q7FOWBzO6xPHPrSPYWV3L/8i006NrRysdoYvByJRU1bD50gimD9TaSL7nykiQevj6T1buKefy93XaHo1Sb6KgkL7d6VxHGwORMvY3ka2Zflk5BcSWLPt5Pv6Qobh7dy+6QlPKIXjF4uWyni57xEQxKjbE7FHUBfvfdTCb0T+Q3b+5k/f4yu8NRyiOaGLxYRXUdnxWUMSWzGyK69oIvCnYE8cytI+kZH8ndr27mYJlO1a28nyYGL/Zhfgm1DY06aZ6Pi4sIYfHs0QDcuWSTzsaqvJ4mBi+Wk1dEQlQoo3p3tTsUdZHSE6N49tZRHCw7w/zXv6C+QWdjVd5LE4OXqqlvYO3uYq4dlIJDl/D0C+P7JfDvNw7hk72l/Ns7eXaHo9Q56agkL/X5vjIqa+qZMkSHqfqTGWN6UVBcyV8/PUD/lBhmjettd0hKfYNeMXipbGcRUaEOLuuXaHcoqp09dN0grh6QxB9WOXWkkvJKmhi8UGOj4f28Iq4akEx4iMPucFQ7cwQJT80cQa+ESO557QsKT5yxOySlzqKJwQttOXyC0soaJuvTzn4rNjyEF27Poq6+kbmvbKaqtsHukJT6miYGL5TjLCLEIVw9UNde8Gf9kqJZMHMEu1yn+OUb23RpUOU1NDF4GWMM2U4X4/omEBseYnc4qoNdPTCZX04ZwDvbj/HsR/vsDkcpQBOD19lbXMmXZWd0iu0A8pMr+3H90FT+lJ3P2t3FdoejlCYGb5O90wXAJF2UJ2CICH+aPozM1FjuW7aFfSWVdoekApxHiUFEpopIvogUiMiDLewPE5EV1v4NIpLeZN9D1vZ8EZnSWp0i8qKIbBOR7SLyhohEX1wTfUtOXhEjenUhJTbc7lBUJ4oIdbDo9ixCg4P40Su5nKrWaTOUfVpNDCLiAJ4Bvg1kAjNFJLNZsTnACWNMf+BJ4HHr2ExgBjAYmAosFBFHK3X+zBgzzBgzFDgEzL/INvqMIyer2HGkXKfYDlA9ukSw8NaRHCo7w/3Lt+oCP8o2nlwxjAEKjDH7jTG1wHJgWrMy04CXrddvABPFPR3oNGC5MabGGHMAKLDqO2edxphTANbxEUDA/O943+m+jaSL8gSusX0T+N0Ng/lgdzH/lZNvdzgqQHmSGHoAh5u8L7S2tVjGGFMPlAMJ5zn2vHWKyEuACxgI/HdLQYnIXBHJFZHckpISD5rh/bKdRfRPjqZvUkDdPVPN3Da2FzPH9GLhh/t4Z/tRu8NRAcgrO5+NMXcA3YFdwM3nKLPIGJNljMlKSkrq1Pg6wonTtWz88rheLShEhD/cMJhRvbvyqze2k++qsDskFWA8SQxHgJ5N3qdZ21osIyLBQBxQdp5jW63TGNOA+xbTTR7E6PPW7C6modFo/4ICIDQ4iIW3jiQqLJgfL83VNRxUp/IkMWwCMkSkj4iE4u5MXtWszCpgtvV6OvCBcT/GuQqYYY1a6gNkABvPVae49Yev+xhuAAJiJfVsp4tuseEMTYuzOxTlJVJiw3n21pEUnqjigRVbadTOaNVJWk0MVp/BfCAb962dlcYYp4g8IiI3WMVeBBJEpAB4AHjQOtYJrATygPeAecaYhnPVCQjwsojsAHYAqcAj7dZaL1VV28Ane0uYPDhFl/BUZ8lKj+fh72ayZncxCz7Ya3c4KkB4tB6DMeZd4N1m2x5u8roa+ME5jn0UeNTDOhuByz2JyZ98tKeE6rpGfdpZtWjWuN5sO1zOX1bv5dIecUwcpP1QqmN5ZedzoMnJcxEXEcKYPvF2h6K8kIjw6PeGMKRHLPev2MqB0tN2h6T8nCYGm9U1NLJmVzETByYT4tDToVoWHuLgudtGERwk/HhpLqdr6u0OSfkx/Say2aYDxymvqmOy3kZSrUjrGsl/zxxJQXElv/rbdp2mW3UYTQw2y3a6CAsO4opLdAlP1boJGYn8aupA/rn9GC98st/ucJSf0sRgI2MMOXlFXHFJEpGhHo0DUIofX9GX6y7txmP/u5vPCkrtDkf5IU0MNtpxpJxj5dVM1im2VRuICH+cPox+SdHMX7aFIyer7A5J+RlNDDbKcRYRJHCtDj9UbRQdFsxzs0ZRV9/I3Us3U12na0ar9qOJwUbZThdj+sTTNSrU7lCUD+qXFM0TNw9nx5FyfvvmTu2MVu1GE4NN9pdUsre4Uh9qUxdlUmYK913Tn//ZXMhrGw7ZHY7yE5oYbJKTVwToEp7q4v302ku4akASf3jbyeaDJ+wOR/kBTQw2yXG6GNIjlrSukXaHonycI0h46uYRpMZF8JNXN1NcUW13SMrHaWKwQfGpar44dFKn2FbtJi4yhOdnjaKiup55r31BbX2j3SEpH6aJwQbv73LfRtL+BdWeBqXG8thNl7LpyxP8x7u77A5H+TB9qsoG2c4ieidEckmKLuGp2te04T3YXljOi58eYGhaHN8fmWZ3SMoH6RVDJztVXcfn+0qZMribrr2gOsRD3x7IuL7xPPT3Hew8Um53OMoHaWLoZGt3F1PXYPRpZ9Vhgh1BPH3LSLpGhnL3q5s5cbrW7pCUj9HE0MlynEUkRocxoldXu0NRfiwxOoznZo2i+FQN9y3fQoMuC6raQBNDJ6qua+DD/GImZSbjCNLbSKpjDe/ZhUemDeaTvaX8V06+3eEoH6KJoRN9tq+U07UNuvaC6jQzxvRi5pieLPxwH+/tPGZ3OMpHaGLoRDnOIqLDgrmsX4LdoagA8vsbBjOsZxd+vnIbBcUVdoejfIAmhk7S0Gh4P6+IqwYkERbssDscFUDCgh08d9tIIkIdzF26mYrqOrtDUl5OE0Mn+eLQCcpO1+ptJGWL1LgInr5lJAfLzvDzldto1M5odR6aGDpJ9k4XoY4grh6QZHcoKkCN65vA/7tuEDl5RTz70T67w1FeTBNDJ/hqCc/L+icQEx5idzgqgN15eTrThnfnzzn5fLSnxO5wlJfSxNAJdrsqOHT8jE6ap2wnIvzn9y9lQEoM9y3bwuHjZ+wOSXkhTQydIMdZhAhcm5lsdyhKERkazPOzRmGMYe7SzVTV6rKg6myaGDpBttPFyF5dSY4JtzsUpQDonRDFUzNHsNt1iof+vl2XBVVn0cTQwQ4fP0PesVNMGaxzIynvcvWAZB649hLe3HqUJZ99aXc4yotoYuhgXy3hqf0LyhvNu7o/1w5K4dF/7mLD/jK7w1FeQhNDB8txuhiQEkN6YpTdoSj1DUFBwhM3D6NXfCTzXt+Cq1yXBVWaGDpUWWUNm748zmS9jaS8WGy4e1nQqtp6fvLaZmrqtTM60HmUGERkqojki0iBiDzYwv4wEVlh7d8gIulN9j1kbc8XkSmt1Skir1nbd4rIYhHx2YH/a3YX02h0CU/l/TJSYvjzD4ax5dBJHnk7z+5wlM1aTQwi4gCeAb4NZAIzRSSzWbE5wAljTH/gSeBx69hMYAYwGJgKLBQRRyt1vgYMBC4FIoC7LqqFNspxuujRJYLB3WPtDkWpVn370lTuvrIfr204xMpNh+0OR9nIkyuGMUCBMWa/MaYWWA5Ma1ZmGvCy9foNYKK4162cBiw3xtQYYw4ABVZ956zTGPOusQAbAZ9ctPZ0TT0f7y1lUmaKLuGpfMYvJl/ChP6J/OatnWwvPGl3OMomniSGHkDTXx8KrW0tljHG1APlQMJ5jm21TusW0izgvZaCEpG5IpIrIrklJd73aP/He0qorW/U/gXlU4IdQSyYOYKk6DDuXrqZssoau0NSNvDmzueFwMfGmE9a2mmMWWSMyTLGZCUled/EdNlOF10iQxiTHm93KEq1SXxUKM/PGkXZ6Vrmv76FuoZGu0NSncyTxHAE6NnkfZq1rcUyIhIMxAFl5zn2vHWKyO+AJOABTxrhbeoaGlmzu5iJA1MIdnhz7lWqZUN6xPGf37+Uz/eX8eg/d9kdjupknnxrbQIyRKSPiITi7kxe1azMKmC29Xo68IHVR7AKmGGNWuoDZODuNzhnnSJyFzAFmGmM8clfVdbvL6Oiul6fdlY+7fsj07hrQh+WfPYlKzYdsjsc1YmCWytgjKkXkflANuAAFhtjnCLyCJBrjFkFvAgsFZEC4DjuL3qsciuBPKAemGeMaQBoqU7rI58DDgKfW522fzfGPNJuLe4EOc4iwkOC+FaG993iUqotHvz2QPKLKvjNmzvplxRNlt4aDQjiD5NnZWVlmdzcXLvDAKCx0TD+sTUM79mF52dl2R2OUhet/EwdNy5cR0V1HW/Nn0CPLhF2h6TaiYhsNsZ844tKb4C3s+1Hyik6VaNzIym/ERcZwgu3Z1FT18jcV3J1mu4AoImhnWU7XTiChImDdO0F5T/6J0ezYOYI8o6d4pdvbNNpuv2cJoZ2luN0MbZPPF0iQ+0ORal2dfXAZH49dSDvbD/Gwg91zWh/pomhHRUUV7Kv5LTOjaT81o+v6MuNw7vzp+x83remlFf+RxNDO8rJcwEwKVOHqSr/JCI8dtNQhqbFcf/yLewpqrA7JNUBNDG0o2xnEUPT4uiuozaUHwsPcfD8rFFEhgVz18u5nDhda3dIqp1pYmgnrvJqth0+qbeRVEBIjYvg+VmjcJVXM3/ZFzpthp/RxNBO3rduI03W20gqQIzs1ZX/+P6lrCso43ernDpSyY+0+uSz8kxOXhF9E6PonxxtdyhKdZrpo9IoKK7kuY/20Tcxiru+1dfukFQ70CuGdlB+po7P95UxabCuvaACz6+mDGDq4G48+u4uVutIJb+giaEdrM0vpr7RaP+CCkhBQcKTNw/n0h5x3Ld8C86j5XaHpC6SJoZ2kO10kRwTxvC0LnaHopQtIkId/PX2LOIiQpizJJeiU9V2h6QugiaGi1Rd18CH+SVMykwhKEhvI6nAlRwbzouzR1NRXceclzdxprbe7pDUBdLEcJE+3VtKVV0Dk/U2klJkdo/lv28ZQd7RU9y/fCuNjTpSyRdpYrhI2U4XMWHBjO+bYHcoSnmFawam8JvvZJKTV8Tj7+22Oxx1AXS46kWob2hk9a4irh6YTGiw5lilvnLH5ekcKD3N8x/vJz0xipljetkdkmoDTQwXIffgCU6cqdPRSEo1IyL87ruZHDp+ht+8uZNuseFcPVCnovcV+mvuRchxFhEaHMSVA3QJT6WaC3YE8cytIxnYLYZ7XvuC7YUn7Q5JeUgTwwUyxpDtdDGhfyLRYXrhpVRLosOCeemHo4mPCuXOJZs4VHbG7pCUBzQxXKC8Y6c4crJK50ZSqhXJseG8fOcY6hsNs1/aSFlljd0hqVZoYrhA2c4iROBaTQxKtap/cjR/vT2LoyermPOyrhvt7TQxXKAcp4us3l1JjA6zOxSlfEJWejxPzRjBtsKT3LtsC/U6VbfX0sRwAQ6VnWG3q0JHIynVRlOHdOP33x3M6l1FOlW3F9Ne0wuQ8/XaC5oYlGqr2Zelc7S8iuc/2k9KbDj3TcywOyTVjCaGC5DtdDGwWwy9EiLtDkUpn/TrKQMpqajhiff30CUyhNvHp9sdkmpCE0MblVbWkHvwBPddo7/lKHWhgoKEP940lFNV9Tz8lpPY8BBuHNHD7rCURfsY2mh1XhHGwOTBOhpJqYsR7Aji6VtGML5vAj//n22s2aWL/HgLTQxtlJNXRFrXCDJTY+0ORSmfFx7i4IXZWQzuHss9r33Bhv1ldoek0MTQJpU19Xy6t5TJmd10CU+l2kl0WDBL7hhDWtcI7no5l51HdAU4u2liaIOP8kuobWhkit5GUqpdxUeF8updY4mNCGH24o3sK6m0O6SA5lFiEJGpIpIvIgUi8mAL+8NEZIW1f4OIpDfZ95C1PV9EprRWp4jMt7YZEUm8uOa1r2yni/ioULLS4+0ORSm/kxoXwat3jUUEbnlhPV+WnrY7pIDVamIQEQfwDPBtIBOYKSKZzYrNAU4YY/oDTwKPW8dmAjOAwSpkEcoAAA7KSURBVMBUYKGIOFqpcx1wLXDwItvWrmrrG1m7u5hrByXj0CU8leoQfRKjeO2ucdQ1GG55YT2Hj+uke3bw5IphDFBgjNlvjKkFlgPTmpWZBrxsvX4DmCjum/DTgOXGmBpjzAGgwKrvnHUaY7YYY768yHa1u8/3l1FRU68PtSnVwQZ0i+HVOWM5U9fAjEXrKTyhyaGzeZIYegCHm7wvtLa1WMYYUw+UAwnnOdaTOs9LROaKSK6I5JaUlLTl0AuS7XQRGepgQoZX3d1Syi9ldo/l1Tljqaiu45YXNnCsvMrukAKKz3Y+G2MWGWOyjDFZSUkdu1BOY6Ph/bwirrwkifAQR4d+llLKbUiPOJbOGcuJ07XMXLSeolPVdocUMDxJDEeAnk3ep1nbWiwjIsFAHFB2nmM9qdNrbDl8kpKKGp00T6lONqxnF5bcOYaSihpmvrCeYk0OncKTxLAJyBCRPiISirszeVWzMquA2dbr6cAHxj1t4ipghjVqqQ+QAWz0sE6vkZPnIjhIuHqArlmrVGcb1bsrS+4cg6u8mpsXrefoSb2t1NFaTQxWn8F8IBvYBaw0xjhF5BERucEq9iKQICIFwAPAg9axTmAlkAe8B8wzxjScq04AEblPRApxX0VsF5G/tl9z284YQ46ziPH9EoiLDLEzFKUC1uj0eJbOGUtpRQ0/eO5zXSK0g4k/zIeelZVlcnNzO6TuvUUVTHryY/5t2mBm6QyQStlqR2E5sxZvICw4iNfuGkf/5Gi7Q/JpIrLZGJPVfLvPdj53lmyne+2FSTpMVSnbXZoWx/K542hoNMxY9Dm7XafsDskvaWJoRU5eEcN6dqFbXLjdoSilgIHdYlnx4/EEBwUxY9F6theetDskv6OJ4TyOnqxie2G5zo2klJfplxTNyh+PJzosmFte2MC6glK7Q/IrmhjO4/089/zw+rSzUt6nV0Ikb9x9GT26RPDDlzayattRu0PyG5oYziPb6aJfUpR2cCnlpbrFhbPyx+MZ0bMr9y3bwkvrDtgdkl/QxHAOJ8/UsuHAcSbrQ21KebW4yBBemTOGKYNT+MPbeTz+3m78YbSlnTQxnMOaXcU0NBp92lkpHxAe4mDhraO4ZWwvnv1wH7/4n+3U1jfaHZbPCrY7AG+Vk+ciJTaMoT3i7A5FKeUBR5Dw6I1DSIkJ58nVeyg8cYbnbhtF16hQu0PzOXrF0IKq2gY+2lPC5MxuBOnaC0r5DBHhp9dm8Jebh7Pl0Em+t3CdrgZ3ATQxtOCTvSVU1zXqbSSlfNSNI3qwbO5YKqrr+d4z63Q4axtpYmhBtrOI2PBgxvbVJTyV8lWjesfz5rzL6RYXzu2LN/Lq+oPaKe0hTQzN1Dc0smZ3ERMHpRDi0L8epXxZz/hI/vaTy7giI5HfvLmTX/9tO9V1DXaH5fX0m6+ZjV8e5+SZOiZn6tPOSvmDmPAQ/jp7NPdd05+VuYVMf+4zXUu6FZoYmslxFhEWHMSVAzp2VTilVOdxBAkPTB7Ai7OzOFh2hu8+/Skf7en4JYF9lSaGJtxrL7j4VkYikaE6klcpfzNxUApvz59At9hwfvjSRp7Iyae+QZ93aE4TQxM7j5ziaHm1Pu2slB9LT4ziH/dczk0j01jwQQE3L1qvt5aa0cTQRE6eiyCBiQN1CU+l/FlEqIM//2AYC2aOYI+rgusWfMLbOgnf1zQxNJHtdDE6PZ6E6DC7Q1FKdYIbhnXn3Z9+i4zkaO5dtoWfr9zGqeo6u8OynSYGy4HS0+wpqtTbSEoFmJ7xkaz88XjuvaY//9hSyOQnPmbt7mK7w7KVJgZLjrWEpw5TVSrwBDuC+PnkAfzjnsuJjQjmjiWb+PnKbZSfCcyrB00Mlpy8IjJTY+kZH2l3KEopmwzr2YW3753Avdf0582tR5j05Ee8ve1owD0xrYkBKK6o5otDJ3RuJKUUYcEOfj55AG/Nu5zk2DDuXbaFW/+6gYLiCrtD6zSaGIDVecUYA5N1bWellGVIjzjemjeBf7txCDuPlDP1L5/wn+/uorKm3u7QOpwmBtyjkXrFRzKwW4zdoSilvIgjSJg1rjdrf3EVN41M4/mP93PlH9fy8mdf+vVCQAGfGCqq6/hsXymTM1MQ0bUXlFLflBAdxuPTh/LWvMvJSInmd6ucXPvER7y19QiNjf7X/xDwiWFtfgl1DYYpQ7R/QSl1fsN6dmHZj8ax5I7RRIUF89PlW7luwSes2naUBj9KEAGfGHKcLhKiQhnZq6vdoSilfICIcNWAZP557wT+cvNw6hoauW/ZFib+14cs33iImnrfn9Y7oBNDTX0DH+aXMCkzBYcu4amUaoOgIOHGET14/2dX8txtI4kJD+HBv+/g8sfW8l85+Rw9WWV3iBcsoKcQ/WxfGZU19ToaSSl1wYKChKlDUpkyuBuf7C3llc+/5Om1BTyztoBJmSnMGNOLCf0TfWrhr4BODDlOF1GhDi7rl2h3KEopHyciXHFJEldcksTh42d4feMhVmw6TLaziISoUL4zNJVpw7szsldXrx/oEtCJISkmnB9k9SQ8xGF3KEopP9IzPpJfTx3I/ddm8PGeUt7ceoQVmw7zyucHSYkN4+oByVw1IJkJGYlEh3nf17B48qi3iEwFngIcwF+NMY812x8GvAKMAsqAm40xX1r7HgLmAA3AfcaY7PPVKSJ9gOVAArAZmGWMqT1ffFlZWSY3N9fDJiulVOerrKknx+li9a4iPtlTSkVNPSEOYVhaF0ald2VUr66M6t21U2d3FpHNxpisb2xvLTGIiAPYA0wCCoFNwExjTF6TMvcAQ40xd4vIDOB7xpibRSQTWAaMAboDq4FLrMNarFNEVgJ/N8YsF5HngG3GmGfPF6MmBqWUL6lraGTzwROszS9m44Hj7DxSTl2D+7s4OSaMAd1iyEiOoW9SFN27hNMtNoLUuHDiIkIIaseBMudKDJ5cw4wBCowx+62KlgPTgLwmZaYBv7devwE8Le6baNOA5caYGuCAiBRY9dFSnSKyC7gGuMUq87JV73kTg1JK+ZIQRxDj+iYwrm8CANV1Dew4Us6WQyfId1Wyp6iC1zcepLrum09XR4U6iAoLJiosGEeQ8OLsLHonRLVrfJ4khh7A4SbvC4Gx5ypjjKkXkXLct4J6AOubHdvDet1SnQnASWNMfQvlzyIic4G5AL169fKgGUop5Z3CQxyMTo9ndHr819saGw1FFdUcK6/GVe7+s7yqjtM19e6f2gYaGhsJC27/PlLv6/XwkDFmEbAI3LeSbA5HKaXaVVCQkBoXQWpcROd/tgdljgA9m7xPs7a1WEZEgoE43J3Q5zr2XNvLgC5WHef6LKWUUh3Ik8SwCcgQkT4iEgrMAFY1K7MKmG29ng58YNy92quAGSISZo02ygA2nqtO65i1Vh1Ydb514c1TSinVVq3eSrL6DOYD2biHli42xjhF5BEg1xizCngRWGp1Lh/H/UWPVW4l7o7qemCeMaYBoKU6rY/8NbBcRP4d2GLVrZRSqpN49ByDt9Phqkop1XbnGq7qO5N3KKWU6hSaGJRSSp1FE4NSSqmzaGJQSil1Fr/ofBaREuDgBR6eCJS2Yzi+QNscGLTNgeFi2tzbGJPUfKNfJIaLISK5LfXK+zNtc2DQNgeGjmiz3kpSSil1Fk0MSimlzqKJwZqIL8BomwODtjkwtHubA76PQSml1Nn0ikEppdRZNDEopZQ6S0AnBhGZKiL5IlIgIg/aHU97EJGeIrJWRPJExCkiP7W2x4vI+yKy1/qzq7VdRGSB9XewXURG2tuCCyciDhHZIiLvWO/7iMgGq20rrCnesaaBX2Ft3yAi6XbGfaFEpIuIvCEiu0Vkl4iM9/fzLCI/s/5d7xSRZSIS7m/nWUQWi0ixiOxssq3N51VEZlvl94rI7JY+61wCNjGIiAN4Bvg2kAnMFJFMe6NqF/XAz40xmcA4YJ7VrgeBNcaYDGCN9R7c7c+wfubi2+tr/xTY1eT948CTxpj+wAlgjrV9DnDC2v6kVc4XPQW8Z4wZCAzD3Xa/Pc8i0gO4D8gyxgzBPWX/DPzvPC8Bpjbb1qbzKiLxwO9wL5k8BvjdV8nEI8aYgPwBxgPZTd4/BDxkd1wd0M63gElAPpBqbUsF8q3XzwMzm5T/upwv/eBe7W8NcA3wDiC4nwYNbn6+ca8DMt56HWyVE7vb0Mb2xgEHmsftz+eZ/1tbPt46b+8AU/zxPAPpwM4LPa/ATOD5JtvPKtfaT8BeMfB//8i+Umht8xvWpfMIYAOQYow5Zu1yASnWa3/5e/gL8Cug0XqfAJw0xtRb75u26+s2W/vLrfK+pA9QArxk3T77q4hE4cfn2RhzBPgzcAg4hvu8bca/z/NX2npeL+p8B3Ji8GsiEg38DbjfGHOq6T7j/hXCb8Ypi8j1QLExZrPdsXSiYGAk8KwxZgRwmv+7vQD45XnuCkzDnRS7A1F885aL3+uM8xrIieEI0LPJ+zRrm88TkRDcSeE1Y8zfrc1FIpJq7U8Fiq3t/vD3cDlwg4h8CSzHfTvpKaCLiHy1fG3Tdn3dZmt/HFDWmQG3g0Kg0BizwXr/Bu5E4c/n+VrggDGmxBhTB/wd97n35/P8lbae14s634GcGDYBGdaIhlDcnVirbI7poomI4F4ne5cx5okmu1YBX41MmI277+Gr7bdboxvGAeVNLll9gjHmIWNMmjEmHfd5/MAYcyuwFphuFWve5q/+LqZb5X3qN2tjjAs4LCIDrE0Tca+t7rfnGfctpHEiEmn9O/+qzX57npto63nNBiaLSFfrSmuytc0zdney2NzBcx2wB9gH/Kvd8bRTmybgvszcDmy1fq7DfW91DbAXWA3EW+UF9+isfcAO3CM+bG/HRbT/KuAd63VfYCNQAPwPEGZtD7feF1j7+9od9wW2dTiQa53rN4Gu/n6egT8Au4GdwFIgzN/OM7AMdx9KHe4rwzkXcl6BO622FwB3tCUGnRJDKaXUWQL5VpJSSqkWaGJQSil1Fk0MSimlzqKJQSml1Fk0MSillDqLJgallFJn0cSglFLqLP8ff56Mt4kql/IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"Learning rate schedule:\")\n",
    "rng = [i for i in range(1000)]\n",
    "plt.plot(rng, [lr_schedule(x) for x in rng])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}