PTS + PTQ Flow.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d7d91729",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.ao import sparsity\n",
    "import torch.quantization as tq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7608ee79",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model(\n",
      "  (quant): QuantStub()\n",
      "  (seq): Sequential(\n",
      "    (0): Linear(in_features=16, out_features=16, bias=True)\n",
      "  )\n",
      "  (linear): Linear(in_features=16, out_features=16, bias=True)\n",
      "  (dequant): DeQuantStub()\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "from models import Model\n",
    "model = Model()\n",
    "model.qconfig = tq.get_default_qconfig()\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cec36dd2",
   "metadata": {},
   "source": [
    "## Create sparsifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3cbb38ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WeightNormSparsifier (\n",
      "\tModule Group 0\n",
      "\t    module: Linear(in_features=16, out_features=16, bias=True)\n",
      "\t    path: linear\n",
      "\t    sparsity_level: 0.5\n",
      "\t    sparsity_pattern: (1, 4)\n",
      "\t    zeros_per_block: 4\n",
      "\n",
      "\tModule Group 1\n",
      "\t    module: Linear(in_features=16, out_features=16, bias=True)\n",
      "\t    path: seq.0\n",
      "\t    sparsity_level: 0.5\n",
      "\t    sparsity_pattern: (1, 4)\n",
      "\t    zeros_per_block: 4\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "sparsifier = sparsity.WeightNormSparsifier(model, config=None)\n",
    "print(sparsifier)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "068d5970",
   "metadata": {},
   "source": [
    "## Create a scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dbde6350",
   "metadata": {},
   "outputs": [],
   "source": [
    "sl_lambdas = [\n",
    "    (lambda epoch: (1.0 / (1.0 + np.exp(-(epoch / 10.0))) * 2 - 1.0)),\n",
    "    (lambda epoch: (np.log10(epoch + 1.0) / 2.0 if epoch < 100 else 1.0))\n",
    "]\n",
    "\n",
    "scheduler = sparsity.LambdaSL(sparsifier, sl_lambdas)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a933b61",
   "metadata": {},
   "source": [
    "### Scheduler profiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9e88b29a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Sparsity level Scheduling (target = 0.9)')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk0AAAHYCAYAAABQudw1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB1uklEQVR4nO3de3xbd30//pd8iXO1j+1cmubqo97pJZbsAqVQqKW2g43RVrLLLt8xqKWV7bvfNqhEuu3Ll42RyC3bd/vuC5GcDgYMiCUKgwFrpaRQoKWNpKT03kZyk7TpLZaO5SSO48v5/aGcEx9dbN0l26/n4+FHos85Ouej4yPp7c/n/fl8dLIsyyAiIiKiOdVVuwJERERECwGDJiIiIqIcMGgiIiIiygGDJiIiIqIcMGgiIiIiygGDJiIiIqIcMGgiIiIiygGDJiIiIqIcMGgiIqKyCAQCCIfD1a4GLVHluP8YNBERUclZrVaYzWYEg8FqV4WWKK/XC6PRCKfTWbJjMmjKwOfzwWg0QqfTobW1FWazGYFAAB6PB2azudrVqyiPx4PW1tYF99diIBCATqfT/LS2tkKSpGpXrSCVfD3lvP8HBgbU+huNxhLVOHeSJGmuYSnu61L8bhbq+ywbq9UKn88Ht9sNm81W7epQBXg8Huj1evW9HQgEynIep9Opfj4ZjUYMDAxk3Ve5/wYGBubcLy8yaVgsFhmAbDAYZIfDIdtsNlkURRmADEAWRbHaVawom80mA5C9Xm+1q5KXeDwuu91u2eVyyS6XS/29hkKhaletIJV6PeW+/yORiOxyuWSDwSBX6+PH7Xar97Xb7S76eKX43SzU91kmpby2tDAov3Plc0P5zCjl/RyJRGRBEGQAssVikR0Oh/o5YrFY5nyuw+EoWX0YNM3i9XplALLL5UrbZjKZlmTQJMvJmzXX/QRBqMnAxO12L+igKVU5Xk8l73/lQ7ZaQqFQ2b7YC/3d5Po+q2XKdbXZbPPuW8ufF5WyGK6B3++XAcgOh0NTrnxmxOPxkpxHFMWM1yrXgMhgMMiCIBRdH3bPzeL3+yEIAhwOR9o2r9dbhRrVBlEUc9pPkiRIkoRoNFrmGlE58P6vrlzfZ7XM6XRCEAS4XK559+XnxeK4Bk6nE6Iopv3Olc+MXbt2FX0On8+HaDQKl8sFg8Gg2eZyuSAIwrznGRwchCRJRdeHQdMssVgs6zZBENJ+WXOpdO7MQs3VodpRyvufFoZSfm5Eo1EEAgHYbDYIglCy45ZLoa+dn7UXhMNhhMPhjInWgiDAYrGUJJfo4MGDAIDe3t6M2202G8Lh8Jy/G4PBAIPBAI/HU1RdGDTNYjabIUlS1l+y1+uF2+1WH89OatXr9QiHw5oEWr1eD5/Pl/FYs/dVnm+1WrP+xdHa2pqWwCpJEqxWq3o+nU6Xse4ej0dzrtbWVtjtduj1+ozJena7PS2xda7XkZrYq9QpU8Lt7P1Tt/l8vppJ3B4YGFCvmV6vh91uT6tLpgRg5feX62vJ5TyVku/9nyocDsNqtWru1fkSQiVJgt1uV59jNpvn/as732sWDodhNpvV96TT6cwYIGZLUi9H8jiQ3/vM5/NBr9ernxP5XLdAIKC5XkpSbGtrq/pTipYO5fc812CBfD8vUp9bqc/MQq9ZLvdmMdcgHwMDA7Db7fP+ZLvncqGMjMwWzPT19QFA0a9FudbZgnG9Xq+pTzZ9fX2QJKm4+hTVubcIKQlsJpNJ9nq9c/Z/pia14nwinNvtlt1ut1qeKUdEOY/D4ZBdLpfscDjUJLdMfbNer1d2uVxqLojNZpMFQZAFQZAdDoeaj2IwGDTPU5JSTSaTei6lLFvdQqGQmtSq9Ben9lfPpiTBKnWzWCzq810uV1reyOxrM3tbPB6XXS6XWt9S5pvkmmcSj8fVuinXTLlegiCk5Z243W61rz2f15LveQp9PfnK5/7PVB9BENTfv3LvZPqYUe6V2few8h4QBCHjOQq5ZsrvQBAE2Wazqe8bpV6zfy9zJannkzye6+8mn/dZKBTSJOXnet1mfy6kvvdtNptaV7/fP2ddc6Ecez75fl4oKvWZWcg1K+Rzo5BrkI/ZAzjm+pkviXou8+UmKjluxSZgK+/jbJ+Lyvb5rlspchkZNKWIx+OaNwnOJ7/abLasv7DZN36qbMlwbrc74wekkqyWjfJLV97Us48biUTSHme7QeZK+p0tHo/PGzSl1i2XN4iSPJh6TZXyUgcDuX6RKb+v1C8R5Vpm+h0rx079Hc/1Wgo5TyGvJ1+F3P/K791kMqVtU77cUinvGVEUNddtri/xfK9ZtnrF43H1CyXTeyPbF0GuH7iF/G5yfZ/le90EQUgLCpQvmFKP1DOZTHN+dqXK9wu1Ep+ZslzYNSv0/VyqoKJa5vudK/f1fN8z81GuU7brONf7eTbl95HL91k2DJqyiMfjstfrTRtynenDXPkgy/Slku8vab4P3NkfAPO1Rig3bLaRLCaTKaeWl3IETbIsq3/xzab8NVhquXyRKfXP9gZXWgNSP2SVa5T6hlXunVKdJ9/XU4x87n/lCyMfynsm9XjZApNCrtlco3eU67dQg6Zcrttc9Z3rc6FQBoMhr9GVpQoYSvmZWcg1K+b9vNiDJuX7r9igSZZlTYuh8nv0+/2anp753pv5fJ9l0wDKSElis1gsAJJ9sj09PbBarYjH4xmfk2n0iyiKEAQhax9qIBCA3+9HOBxGLBZT+27nSsoFAIvFMu9oG2UUi9PpxNDQELq6umAwGNDd3Q2TyQS/3z/n88vNZrPB4/FoRl0MDQ1VbTK82bk3mfr5ldyEaDSqSYpWkqS9Xq+m7tleS6HnqaR87n8l+bcQXV1dOe1XyDULBoMwmUwZ8yByPW+tqsX6i6JYVH5Mrsr5mVlofRS19H4eGBhAJBKZdz+z2ay+z/MliuKcOYvK76YU113Jp/R4PJpkboPBoH7PtbW1zXkM5R5pb28vuB4MmmZRAptMN7bBYMDg4KA6020+N5koimlJg4FAAHa7HdFoFKIoqpn90Wg0pw+e7u7unM7tcDhgsVjgdrsRDofh8XjUxEeLxVLVoeR9fX0YGBhAIBCAyWRSEzWV5MFKGxkZAYCCptxXkiolSYIgCAgEAllfSzHnKadC7n/lvlYSMfOV6yirQq6ZJEmLYhh/JrlcN4PBAEEQ0mblVr5wSr26gXKtlfdAqVXiM7OQa1ar7+ddu3blNKgkFosVHDQpiezhcDjj50YpgyYgGTg5nU41YFYaApRrP19QqnzGFVMfjp6bxel0oqenJ+t2k8kEAHmPNEn9C0MZzSMIAkKhECKRCLxeL1wuF+x2e07HzPVDyePxoK2tDS6XC36/H/F4HJFIBG63Wx0hUgmZrpnBYIAoimrgtm/fvqoObVe++EOhEORk13XGn0z1U0aPKH91eb3erK+lmPOUUyH3v/LhowwJLpdCrpkgCFlbH+ZrlVgsent71VFns0dTmUymgr8os1ECiqGhoaKPlfp5UcnPzHyvWbnez8WOaIzH43PWR/kp5g9npcVz3759Gbcrxy7lZ5koirBYLLDZbOpxfT4fRFGcNxhSeleUz7JCMGhKMddEY/NFzZmaKZUWh9l/5Sg3mNfrTbuZch1uPl8zJJD8oLHb7WmTeYmiCJvNBpfLNe/cFoVI/UKyWq1ZgzO73a7+Fefz+bIOXa0E5Y0017B6p9OZ8XopAZLy3Lm6GYs5T7kVcv8bDAb1Ps+k2HlRgMKuWVdXV9Z6FfJFsRAnIBwaGlK/6J1OJwKBgPoHVKkpXaH5XttcPi8q9ZkJ5H/NSvF+zuczs5YorX2ZWvqUebsyTZZbCI/Hk/E9qJTPdf0VQ0NDWbvsc8WgKQNlHpRUymy32aLU1Dk5otEo7HZ72izLSn9qphugHB9mAwMDGc+l9HeXqildOU5qP3o4HM4aaCqBhVJHq9VakroUQhRFOBwOTRfmbFarNeu1BJLdjYFAYM6uuVKcp9zyvf+VmXYzPc/pdMJutxc9T0sh18zlckGSJPT392v2Vbqps1Hu49Trn8uHcq1R7kOlFSQSiZTsSywTm82mvgfmk8/nRSU/M/O9ZsW8nwv5zKw1LpdL/a6bTfks37lz55zP9/l8MJvNc35GKA0AqV2gPp8PdrsdBoNh3tYjJXAtthuVOU0ZhMNhtLa2wmKxoLu7GyMjI+o07kq3SyaxWAxGoxEWiwWSJGFoaAiSJKW9qW02G3bt2gWz2QybzQa9Xq85B5D8BXd1dcFqtcJkMsHn8yEWiyEUCgFI/uWl/HUiiuK8N4xer9ecKxAIIBwOZ/wwCAQCaj2UL8HZXzTZzqfkGQwMDGhye5TrlonyJTxfQFqI2V+Myu9g3759mgnQUluDlNY3p9OJffv2wWQyob29Hfv27UM4HNY0CaeyWCxwOp2wWq3zdjMWcp5CXk8h8r3/ZydidnR0oLe3F3q9Hn6/X00SV16L8tenUmePx6PeTx6PR/3yUF7f7NeT7zUzGAxwOBzqhIMmk0nNf1HyDL1eLyKRCHbu3Km+LrPZjIGBAVitVvT19aW9N91uN0KhkLp8g/I6FLn+bvJ5nxV63ZT3VjQaVeva1tamvldLzeVywePxwG63z5uEnM/nRSU/Mwu5ZoV+bhTymVlrTCaTOqgnGo3CbDarr1tZmmku/f39ajCTLQBWAshAIKAGPcp3mCiK2L9//5zniEajGBgYgMlkKv47puBxd4uQzWaTHQ6HHIlE0iaSm2t4vjIUUpnjRhAEWRRF2WKxZB3iGo/HNRPtKfuHQiHZ6/Wqw3eVoZqzJ+RL/UmdU0ShLAbp9/vVeimvx2AwZB3mOnsIZz7nU17X7HMZDIZ5J85T5owqZpK1VMocSfP9ZBvW7/V61eG0yu8/l2HByhxHuQ6xzfU8xb6eXBR6/ytCoZBssVjUyT4zTeqpzHcz+0eZcyjT68l0znx/N8r+yrlcLpc6GaEoirIoimnn8Xq96utX3pvK8ObU5xT6u8nnfVbIdZs94WK2n3IsWKwMo0+dEymTfD4vKvGZWew1K+Rzo5DPzFqkTPSrvIZcp9xwuVyyIAjzXqdQKKRe29nv5fko34Opc5sVikFTCVR7xfaFTvmQXYgfFES1SvmCSX1fxeNxdYZxFDlnTTZKIFmqL6pKqeY1o9ILhUJqAFuq+1Any7KcrRWKcqMkM/NSFmZgYABOp5PXj6hEJElCa2srvF7vnKPk7HY7hoaGss49VwxlLUKLxaKZi61W1cI1o9JSRnTOlVaTL+Y0UdW53e6SD38mWspynR9Hr9eXbZSmwWDIaXLFWlEL14xKqxwDqzh6jirOarWqIyuURM75RlgQUe6USRqVJNtMwuEwdu3axT9YzuM1o1ywpakIc41ooezC4bA6RNXj8cw5Io2IChMKhWA0GjUjIQVBQCQSQTgcRiAQUGd6pyReM5oPc5qKoOTizCaK4oJqkq4GZTmEWCymTrJJROXh8XjgdrsRjUbVpWVEUYTdbmeLSRa8ZpQNgyYiIiKiHDCniYiIiCgHzGnKw8zMDE6cOIE1a9ZAp9NVuzpERESUA1mWMTY2hosvvhh1dYW3FzFoysOJEyewZcuWaleDiIiICnD8+HFs3ry54OczaMrDmjVrACQvenNzc5VrQ0RERLlIJBLYsmWL+j1eKAZNeVC65Jqbmxk0ERERLTDFptYwEZyIiIgoBwyaiIiIiHLAoImIiIgoBwyaiIiIiHLAoImIiIgoBwyaiIiIiHLAoImIiIgoBwyaiIiIiHLAoImIiIgoBwyaiIiIiHLAoImIiIgoB1x7joioBsmyjBkZmJyewfSMjKkZ+fy/5x9Py5iRZ5Wffzw9I2NaljEzM/v/SG47Xz4jJx9r/q/8nN9XVsvPP1bqNJP8/4ycfCzP2q48TynXlCVf1PnjADIulCuPoT6Wz1+DlO3n/69cH1m9Vhm2z9qmlFx4bsq1zvLcC7+L9H2RZd86eQrrzx3HprOvYPPZlyFMvp1hL/Vgc5pn84JQ6Gu46M77cXHHFSWtSykwaCIimmV6RsaZc1MYPzeNM+emMT6Z/Dmr/sxgYir579nJaUxMzWDifNm5qZnk4/P/Pzc9g3NTMs5Nz2ByagaT08mfc9MyJqdnMDU9g8nz/08+ljE5k/x3amYxfGUubo2YwmW61/CuumFcoxvG1XWv4grdMazQnat21Ra8I4mRalchIwZNRLQoTM/ISIxPYvT8T+LsJBLjU0icncTY2UmMnZ3C2NkpnJqYwqmzUzh9Lvn/0xNTOD2RDIxOT0xhYmqm2i+FatAyTOJy3XFcUzeMq3VRXFM3jMt1x7FMN13tqlEFMWgiopo0MTWNkVPn8M7YBN4Zm8DI6QmMnD6H2KlzGDl9DvEz5xA/M4n46XOQzpxD4uxUtatMi0QjpnC57hiuPR8gXVs3jMsYIBEYNBFRFZyamMLr8XGcGB3HG9JZvDE6jjdGz+KtxFm8nZjAW2NnIZ2ZrHY1FxSdDmio06FOp0N9nQ71Oh3q6pL/T5YBdbrk/+vqkNx+fh8doO5Xd34/nU6HOh3UbTpdcj+d7vxj6KDTQbPfhX3Ob4NyrOS/SNvvwmMg+f/kPrpZ22Y91qmvVn3NulmvXzer/MJeF46v3Zb8Tx2msH58GBeffgEbz7yIjadfxPrxI2iQi7v/EsvW461Vl+PkChHTdcuy7qfLuiXL/vk+Ye6j1eCRki5dt7nERywNBk1EVHIzMzJel8ZxdOQMjsZO49jIGRwdOYPXpDN4LT6+oAIinQ5Y0ViP5Y31WNFYj6aGOjQ11mN5Yx2WNyT/XdZQh6aG+vP/znpcr8Oy84+X1dehsaEOjfXn/19fh4Z6HRrrdcn/19WhsV6Hhvo6NNTp0FCvQ0Ndyv/rdcnA6HxQVF+XfKwENTSPmRkgFgFeDwMnwsl/3/wNMHW2uOO2bgc27gA2XgdsvBbYuAPNq9aiGcClJag21Q4GTURUsLOT04i+cxqvvD2GV946hSNvn8LwydMYHjmNc1XMDVrWUIeWFY1oXt6A5hWNWN3UgObljVizvAGrmxqwqunCv6ua6rG6qQErlyX/v3JZPVYsa8DKxnqsWJYMkhiQLECyDCROnA+OQsmfE08DE6PFHbf9kmSAdPGOZJB00bXACqEEFaaFgEETEeXknbEJPHtiFM+fSOCFN5I/wydPo9yDvBrqdFi3pgntq5ehfZXy7zK0rlqGtpXLIKxchtaVjRBWLoOwshEtKxqxvLG+vJWi2nM2cSFAeu18kHTqzeKO2aYHLu48HyDtSLYiLW8pRW1pgWLQRERpTk1M4enjEg4di+M3r43imddH8cZokV0YGaxdvQwbW1ZgY8tybGxZjg0ty7FhzXJsaF6O9c1NWLe6CS0rGlFXx5YemmV6Cnj7eeD1IPDa+Z+TL6OomY2EbcAmQzJIUlqSGCBRCgZNRIS3E2fx5HAMTw3HEDwax0tvJkrSgrSmqQFb21diW/tKbG1bhS1tK7CldSU2t67AxcIKtghRbsbeBI4/dSFIOnEImDxT+PFWXwRsMgKbOs+3JBmAlW2lqy8tWgyaiJag0TOTeDxyEr84chJPREYwfPJ0UcfbJKzApRtW49L1q6Fftxoda1dBXLcaa1cvYz4Q5WdqAnjjN8BrB4HXnkoGSaPHCz9eU8v54MhwPlAyAM0Xl66+tKQwaCJaAmRZxnMnEgi88BZ+9tI7+M1rUkEtSSuX1ePKjc24cuOa8/8247INa7C6iR8lVCClFen4k8l/33gamJ4o7Fh1jcBF1wCbu5JB0uauZF5SHZdZpdLgJx3RInVuagaPR07i4efewoEX38Jbify+iBrqdHjXxc3o3NqKaze34JpNLRDXrUY984uoUNNTwNvPzQqSngSkY4Ufr7UjGRht6kr+e9E1QENT6epLlIJBE9Eicm5qBr945R385Jk34X/+zbxmyV7T1ICu7a3o7mhD17Y2XLu5hTlHVJyziWQ32/EngWO/To5oO3eqsGM1NSe71jZ3J382GYFVa0tbX6J5MGgiWuBkWcah4xK+H34d//WbE4jnOHFkU0Mdru9ow/svXYv3imtx1cXNbEWi4iROAMeeSAZIx54A3noOkAucr2vdFcngaMv1yX/XXgbUMYin6mLQRLRAnTw1ge+FXsO+g8cRzTGRe3v7SvRcuQEfvHwdure3sSWJCifLwDsvzQqSHi+8q62pOdlytOX65M+mLk4YSTWJQRPRAiLLMp6IjuA/njyGR557E5PT82dzG7YKuO3qi9Bz5Qbo162uQC1pUZqeAt56Bjj6ePLn2BPAmZHCjtXaAWx9T7IFaet7kq1KbEWiBYBBE9ECcHZyGj88fAL/9qthvPjm2Lz7G7YK+O1rL8ZtV1+Ei4UVFaghLTpTE8n5kF795fnWpCeBc/Pfe2nqGpMTRW55dzJA2vJuYPX6kleXqBIYNBHVMOnMOXz98VfxzSeOYuT0uTn33d6+Erd3bsbHOi/GtvZVFaohLRqTZ5OTR776y+TPawcLW8h2ecuFAGnre5OTRzYycKfFgUETUQ16Z2wCe38ZxbeeOIrT56az7tfUUIePXLsRH79+K7q2tXIiScrd5NlkYDQ7SCpkfqQ1FwPbbkgGSdtuANZdyXmRaNFi0ERUQ2Knz+Erjx7BN399FBNT2UcdiWtX4Q/fuw13dG5Gy8rGCtaQFqypc8mWpOHHkkHS8acKC5LaLwW2vRfYekMySBK2AgzWaYlg0ERUA05NTGHvL6LY+4thnJrIPrfS+y9di0/e2IGbLl3HRWxpbtNTydm1h3+eDJSO/RqYGs/zIDpgw7uSwdG29yX/ZT4SLWEMmoiqaHpGxrefOob/4385a85SnQ74nesuxj0f1OOKi5orXENaMGQZePsFIPqzZJB09FfARCK/Y+jqkrNqb7sR2P6+ZE4SF7IlUjFoIqqSJyIj+MKPnss6Gq6hToc7DZtxzwf12L6Wid2UgXQ8GSQpgdLpt/N7vq4OuOhaYPuNwPb3J/OSOD8SUVYMmogq7M3Rs/i7/3oOP3nmzYzbdTrg9h2b8Jfmy7ClbWWFa0c1bVwCXv0FEHk0GSjFInkeQAdcdDWw/QNAx/uTLUkMkohyxqCJqEJmZmT8x5NH4frvl7LmLZmuXI/P3no5u+EoaXoyuV5b5EDy5/VQ/suSrL0cEG9KtiRtv5HdbURFYNBEVAGvvDWGzz30DEJH4xm3X3HRGvyv37kKN+i5AOmSF4sCR/YnW5OGH8t/QsmWLUDHTclAqeMDwJqLylNPoiWIQRNRGc3MyNj7yygeePhlnJtObyEQVjbiM7dcjo93b0FDPee2WZImTiW73I4EksFSfDi/569ovRAkiR9MLlHCKQCIyoJBE1GZvC6N4zNDh/HraCzjdqtxM+778JVoXbWswjWjqlJGuR0JAEf8wNEngJnJ3J9f35ScJ0n8ICB+KJnIzckkiSqCQRNRGfzn4dfxNz94FmNn03OXtrWvxJduvwbvu4RdcUvGxBgQ/TnwyiPJ1qTEa/k9f8M1gP6DgP7mZPI2lyUhqoqaDpoCgQD8fj/0ej0kSQIAOByOvI/j8/ng9/s1ZS6XC4IglKCWRBecnZzG3/3X8/j2k8fStul0wN03duCvzJdjxTKu6L6oyTJw8pVkkPTKI8DRx/NrTVq5Nhkg6W8G9B9iXhJRjajZoMnn82Hfvn3wer1qWSAQgNlsTguA5uJ0OmE2m+F2u9WyaDSKnp4e7N+/n4ETlczx2Bl8+j/CeOb10bRtm4QVeMB6Hd6rb69CzagipiaSy5O88gjw8n8D8Vdzf25dA7DlPcAlNwP6Hna5EdWomgyaJElCf38/hoe1CZEmkwlOpxMejwc2m23e44TDYfV5s4miiMHBQfT392uCMqJCHXjxLfzFdw8jkaE77vbOTfjC774Lzcu5Rtyic+rtZJD00k+To90mT+f+3JYtwCU9wCXm5Ci35ZxmgqjW1WTQNDQ0BFEUM7YC9fX1we125xQ0BQIBdHd3Z9xmMBjUoIqoULIsY/AXUez66YuQZe225Y11+IePXYM7jZurUzkqPSWJ+6WfJFuTXgsCkOd9GgCgrjGZwH3pLclAad3lHOVGtMDUZNDk9XrR1pZ5AjZRFBEOhyFJUk5da263GxaLJa08Go1CFMViq0pL2LmpGfzND57BUDA9qVdctwpf/X0jLr9oTRVqRiU1PQUcexx48SfJYEk6mvtz12wELjUDl96anBKgifcD0UJWk0FTMBhEb29vxm1KoBONRmEwGOY8jsVigdPphNVqxeDgoCbIcrlccDqdJaszLS3x0+dg/1YITw2nTyfwkWs3wnXntVjdVJNvL8rFudPJUW4v/jjZonRWyvGJOmCTEbjsNuCyW5K5SWxNIlo0avJTPZdWpFyCJlEU1eAoEAjA6/XCZDLB4/HAbDan5ToR5eJ1aRx/+OCTiL6Tnr/iuO1y3HOTHjp+US48p0eAl38KvPBfQPRRYOpsbs9btjo5yu2y25Jdb6vXlbeeRFQ1NRk0zUUJpmKxzBMGpnI4HBBFEVarFWazGaIowu/3s2uOCnLk7TH84YNP4Y1R7RfqisZ6/FPfDtx2NYeGLyijrwMv/hfwwo+Ao7/KfV23li3JIOny30qu59bQVN56ElFNWHBBUyEEQYDD4UA4HFanLfB6vfO2VBHNduhYHH/89YOQzmjn27moeTn2/lEXrt7UUqWaUV5iw8ALPwSe/yHwejD35228Drj8I8lA6aJr2O1GtAQtuKBJmeQyW6J4KqfTie7ubrhcLgCAx+OB3W6H0WiE3++fs4tuYmICExMT6uNEIlF4xWlBe/zISXzq34MYn5zWlF+6fjW+8anrsbGFMzTXtJEI8Nz3gef/E3jzN7k9p64h2Yp0+UeAKz4MtHAUJNFSt+CCpnx4PB4A0Iyes9lsMJlMMJvNsFqtGB4ezpo/tWvXLnzhC1+oRFWphv3qyEl86t8P4uyktuumc6uAr32iG8JKrh1Xk04eAZ7/PvDcD4C3ns3tOY0rk3MnXfE7yUTuFa1lrSIRLSw1GTSJoohoNJpxm5LLlEtOksvlQiQSyXj8UCiEjo4OBAKBjFMSAMDOnTvxV3/1V+rjRCKBLVu25PISaJF4PEvAdNNl6/DVPzBg5bKafAstXbEo8OxD5wOlZ3J7zvIW4LLfAq78nWRC97KVZa0iES1cNfmJbzAY1G64VEowNV8+0nwj8ARBwM6dO7MGZwDQ1NSEpiYmeC5Vjx85iU9mCJg+cs1G/FPfDixr4DIXNWH0tfOB0kPAiUO5PWflWuCKjwBXfRTY/gGgga2FRDS/mgyazGazmoOUKhKJ5DRVgCAIWQOv2TiKjjI5+Gosa8D0z3ftQEM9A6aqOvUO8PwPgGd8wPFf5/acVeuTQdJVvwtsvQGor8mPPyKqYTX5yd/b24tYLJaxFcjn88Fut2vKJElCIBBI21cUxYzlivkSwWlpeuGNBD75dQZMNWdiDDj8HeCbdwBfvhz4yWfnD5hWrQe6+4FP/Bj4zIvAR76cXOeNARMRFaAmP/0FQcDg4GDajN0+nw+iKKblIClzMCmJ3wqv16tObDmbJEmw2+1wOp05LcVCS8fx2Bn80b89hbGUhXcZMFXJ1LnkYrjeTwD3XwL84E+AyH5Ans7+nJXtQNcngT/60flA6YHkKLi6+opVm4gWp5r9c8tisUAQBDidTuj1erWrze/3p+1rNpsRDAbR1dWlKRcEAaFQCE6nE16vV7PN5XIxYCKNk6cm8IcPPom3xyY05aYr1+P/MGCqHFlOLoT7m+8mc5XGc5jIdnlLMpH76juTOUpsSSKiMtDJcura7JRNIpFAS0sLRkdH0dzcXO3qUAmdnphCn+cJPPu6di6u7u2t+Oan3o3ljWylKLv4UeA3+4Cnv5McBTefxpXA5R8GrrEkR71xVm4iyqJU39/8c4yWvJkZGX+x73BawHTFRWuw94+6GTCV08RYcsLJw98Bjv5y/v3rGgB9D3Btb3Jm7mWryl9HIqLzGDTRkjfw8EvwP/+Wpmxz6wp845PXo2VFY5VqtYjNzCQDpMPfTgZMk2fmf86WdwPXWIF33QGsai9/HYmIMmDQREuaN3gce36unQC1ZUUjvvHJ67G+eXmVarVISceSLUqH/wOQjs6/f5sIXPfxZPdbG6cGIaLqY9BES9ZTwzHc933trNENdTp89Q8MENetrlKtFpmpCeDFHwOHvglEHgUwTwrlciGZzH3dx4HNXVwUl4hqCoMmWpLeGB3HPd8KYXJa+yX+9x+7Gjfo11apVovIOy8BoX9PJnXPN/pNVw9cak4GSpf/FhO6iahmMWiiJefc1Az+9D/CGDl9TlP+qRs78PHrt1apVovAuTPJHKXQ13ObpXvdlUDn7wPX9gGr15e9ekRExWLQREvOrp++gPAxSVP2ocvX4b4PX1mdCi10b78IhL6WbFU6Ozr3vk3NyRylzj8ALjaw+42IFhQGTbSk/OjpE/jar17VlG1pW4H/09eJ+jp+geds6hzwwg+B4L8BR381//7bbgQMfwhc+VFg2cry14+IqAwYNNGSceTtMTi/9xtN2bKGOnz1941oWcmpBXIiHU92v4X/HTj9ztz7rloH7Pg9oPN/AGsvqUj1iIjKiUETLQlnJ6fxp/9xCGfOadcs+/vffReu3tRSpVotELIMDP8ceNIDvPxTQJ6ZY2cdoP8QYPwEcNlvAQ3LKlVLIqKyY9BES4Lrv1/ES2+Nacqsxs3o62bid1YTp5J5Sk8NAidfmnvflWuT3W+GPwLaOipTPyKiCmPQRIvez19+Jy2P6YqL1uDvP3Z1dSpU62LDyUDp0DeBicTc+267Eej+JHDF77BViYgWPQZNtKiNnJrAZ71Pa8qaGurwLx/v5Jpys8ky8OovgV9/FXjpJ5hzEspla4Dr7gK6PwWs54hDIlo6GDTRoiXLMj730DN4Z2xCU37fh6/EZRvWVKlWNWbqHPDcQ8AT/w948zdz77v2MuB6WzJgauL1I6Klh0ETLVr7Dh5PW4j3g5evw/9477Yq1aiGjMeB4NeAJ93AqTfn2FGXnKX7ehsgfpDzKhHRksagiRalN0bH8cUfv6Apa1+1DAOWa6Fbyl/80jHgia8A4W8Ak6ez79fUDHT+IXB9PxO7iYjOY9BEi44sy/jr7z+LUxNTmnLXnddi/ZrlVapVlb35DPCrfwaefQiQp7Pv19oBvOee5PxK7IIjItJg0ESLzn8ePoEDL76tKbvTsBmmqzZUqUZVIsvJ2bp/+U/AkcDc+269AXjvnya74uqYIE9ElAmDJlpUTp6awBd+9JymbO3qJvztby+hUV6yDLz8MPDLfwSOP5l9P10dcNXHgBv+DNhkrFj1iIgWKgZNtKh8/ofPIX5mUlP2xY9dDWHlEphDaGYaeP4HwC/+EXjr2ez7Na4CDP8j2Q3XyqR4IqJcMWiiRSPw/Fv48W/e0JR95JqNuO3qi6pUowqZngKe9QG/+DJw8uXs+61sB979J0D33cDKtsrVj4hokWDQRIvC+LlpfP6H2m45YWUj/vdH31WlGlXA9BTwm33AY/cD8eHs+7VsBd7358CO3weWraxc/YiIFhkGTbQofOVnR/C6NK4p+1+/fRXWrWmqUo3KKNdgae1lwI1/BVxjAeobK1c/IqJFikETLXjDJ0/D/fOopuzdHW24vXNTlWpUJjPTwG+GgJ+75g6WNlwD3HRvcj24urrK1Y+IaJFj0EQLmizL+F//+SzOTc+oZQ11Ovz9x65ePJNYzswAz38f+NnuuXOWLjYANzmAy27jzN1ERGXAoIkWtIefexO/eOWkpuyTN3YsjrXlZBl45RFg/98Dbz2Tfb9NRuCDO4FLTAyWiIjKiEETLVhnzk3h7370vKZsQ3MT/rzn0irVqISOPg4EvgAc/3X2fS42AB+6j8ESEVGFMGiiBWvwsWGcGD2rKfubj1yF1U0L+LZ+67lksPTKw9n3ueha4EN/DVx2K4MlIqIKWsDfLrSUvZ04C/djEU3ZDfp2/Pa1G6tUoyKNvgY8+iXg8LcByJn3WXdFMli64reZ4E1EVAUMmmhB+vIjL+PMuQsLz+p0wN/+9lULL/n7bCK53MmvvwpMnc28j7A1GSxdY+W6cEREVcSgiRacF95IYCh0XFPWa9yCKzc2V6lGBZieAkJfS46IO3My8z6r1gE3OQHDHwENS2AZGCKiGsegiRYUWZbxpZ+8AHlWD9bKZfX4zC2XVa9S+XrFDzz818DJlzJvX7YauOHPgff+KdC0urJ1IyKirBg00YLys5ffSZtiwP4BPdY3L69SjfLwzsvAw/cBR/yZt9c1AMY/Ts61tHp9ZetGRETzYtBEC8b0jIxdP3lBU7ahuQn9H+ioUo1ydHY02Q33lAeYmcq8z+UfAcx/B6y9pLJ1IyKinJU9aEokEmhuXkC5JlSzfvj063j5rVOass/ecjlWLqvR2H9mBnj6O0Dg88DpdzLvs3EHcOs/ANtvrGjViIgof2Udtzw4OIjW1tZynoKWiMnpGfyfwCuasis3NuMOw+Yq1WgeJw4B/3YL8J+fzhwwrd4AfOyrQP+jDJiIiBaIGv0TnUjLF3oNR0fOaMo+e8tlqK+rsSkGxiXgwBeBg3uRcb6l+mXAe/8MeP9fAU2LYKkXIqIlhEET1byzk9P4l/3aVqYdWwTcfEUNJUvLMvCMNzkq7vTbmfe5/MPJrrg2sbJ1IyKiksgpaEokEohGo3kfPBKJzL8T0Ty++9QxvJGyXMpnb7m8diayHIkA//WXwPDPM29v0wO/5QIuNVe2XkREVFI5BU3bt2/H6Oho3geXZbl2vthoQTpzbgr/+qg2+H53Rxved0l7lWo0y9Q54PF/AR67P/Ns3g0rgA98FrjhfwINTZWvHxERlVROQZMkSbBarTCZTHkd3O/343vf+15BFSMCgG88cRQnT01oyj57aw20Mr0WBH74P4G3n8+8/bLbkq1LrdsrWi0iIiqfnHOauru70d/fn/cJGDRRoc6cm4LnMW238E2XrUP39rYq1QjAuTPJRO9ffwUZE72bNwEfvh+44iMVrxoREZUXE8GpZn3nqeOInT6nKavqcinDjyVbl+Kvpm/T1QHX24Gb/5qj4oiIFqmcgyZBEPI+uCzLkOUMf40TzWNiahqex7S5TB+8fB2u3SxUvjLnTgP+zwMHBzNv33AN8NF/BjYZK1svIiKqqJyCJr/fj66urrwPbrPZoNfr834e0fdCr+OthDaX6c8+VIUlRo4+AfzgHiA+nL6tvgn44OeSid71jZWvGxERVVROQVNPT0/BJyjmubQ0TU3PYM/P00fMdVUyl2nyLHDg74En/h8y5i5teTfw0X8F1lWxu5CIiCqKOU1Uc370mxM4FtPO/v1nN1ewlenNZ4CHbJlHxjWsAEyfB663AXX1lasTERFVXUmCptmTX+7YsUN9vGPHjlIcnpaQmRkZX0mZl+m6zS248ZK1lTg58MS/JluYps+lb9/y7uR6ce3sciYiWoqKWrD31VdfRV9fH1pbW2E0GtW8py996UswGo349Kc/XZJK0tLxyPNv4pW3T2nK/vRDl5R/XqbECeAbHwX8f5seMNU3Aea/B/74pwyYiIiWsIJbmkZHR2EwGCBJEhwOB+LxOPbu3QsA2L17NyKRCNxuN8xmM26//faSVZgWN3fKvEyXb1gD05UbynvSl34K/ODTwHgsfdtF1wB3DALrryxvHYiIqOYV3NLkdDoxOjqKaDSK3bt345ZbbtFs93q96OzsxJ49e4quJC0NoaNxHDomaco+/SE96urK1Mo0eRb4iQP4zl0ZAiYd8L6/AO7ez4CJiIgAFBE0BQIBWCwWbN++Pes+fX19CAaDhZ6Clpi9v9C2Mm0SVuAj12wsz8lGIsCDJuApd/q25s3AJ34MmL/ANeOIiEhVcNAUjUYhiuKc+4yMjBR6eFpijo2cwcPPvakp+8QN29FQX1TaXWbPfR9w35QcJZfqyo8C9/wS2P6+0p+XiIgWtIJzmgwGA8Lh8Jz7BAKBgibFpKXna48PY2bWdEirmxrQd/2W0p5kagJ45G+Apzzp2xqWA7ftBoyfAKq9GDAREdWkgv+M7+3tRSAQwKOPPppx+/33349Dhw7BbrcXXDlaGkbHJzF08LimrK97C5qXl3CW7dHXgK/9VuaAad0VgO1nQNcfM2AiIqKsCm5pcjgc2LdvH0wmEwYGBjAzMwMAePrpp7Fnzx515Nwdd9xRssrS4vTdp47h9Llp9XGdLtk1VzLDjwHePwbOnEzfdt3HgY98GVi2qnTnIyKiRamoyS0PHDgAi8WCe++9FzqdDrIsw2AwQJZlOBwO7N69u1T1pEVqcnoGX3/8VU3Zb12zEVvaVhZ/cFkGHv+/QODzgDyj3dawHPjwA0DnH7B1iYiIclJU0NTS0gK/349Dhw4hEAhgZGQEer0eJpMJHR0dRVcuEAjA7/dDr9dDkiQAyRauQkSjUTidTgBAW1sbBEGAy+Uquo5UnJ8++ybeGD2rKbv7xuLvHUyOA//5Z8CzvvRtrR1A37eAi64u/jxERLRklGQZlc7OTnR2dpbiUCqfz4d9+/bB6/WqZYFAAGazGX6/P+9j7dq1C16vVx3xJ0kSnE4nA6cq+0ZKK1PXtlZ0bm0t7qCjrwPf/T3gjcPp2y67DbjdDawQijsHEREtOQUngicSiVLWQ0OSJPT392NwcFBTbjKZEIvF4PFkSObNIhwOo7+/H/v379dMkdDf3w+fL0MrBFXM8ycSCB6Na8o+8b7txR30+EFg8EMZAiYd8MH7gLu+w4CJiIgKUnDQ1NHRgQcffLCUdVENDQ1BFEUIgpC2ra+vD253hgkJs3A6ndi5c2fascxmM0f2Vdk3f31U83j9mibc+q6LCj/gMz7g6x8BTr2lLV+2Bvj4d4EPOoG6Msz7RERES0LB3XPxeBxDQ0Po6urCddddV8o6wev1oq2tLeM2URQRDochSVLGoGq2cDiMQCCg6eJT2Gy2UlSVCjQ6PokfHHpdU/bx67eisZDJLGUZeOx+4NF/SN/WJiYDpnWXF1hTIiKipKL+7Pb7/TAYDLj00ktx33334fDhwyWpVDAYzDrbuFIejUYzbp/N7XZnbbGi6vpe6DWMT16YZqChToffe/fW/A80NQH84J7MAZP4weTacQyYiIioBIpKBB8YGEBnZyeGhoawZ88euFwuCIIAu92O3t5e7Nixo6Dj5tKKFI1GYTAY5twnEAioQdbAwAAAQBAERCKRjF12VBkzMzK+ldI1d+u7LsKG5uX5HWhcAvb9AfDqL9K3dd8N3OYC6ksy1oGIiKjwoMnr9cJsNqO5uRk9PT1wu91qV9iePXuwe/dutLa24r777sNnPvOZklVYCXRisdRV6dMpgdXAwIBmqoJoNAqj0YhQKMTAqQp+FTmJ6MnTmrI/eM+2/A6SOAF8607g7edTNuiA23YB7/4Tzr9EREQlVXD33J133onm5mZNmclkgtvtRiwWg9/vVye5rKZwOAyLxaIpE0URJpMJ/f39cz53YmICiURC80PF++YT2lamyzasxnvEzDlsGb39IrDXnB4wNa4E7vo28J57GDAREVHJFRw0vfrqq2lliUQCDz30EG699VbccsstkCQJd955ZzH1S6NMcpktUTyTTPlRRqMRPp9PPV4mu3btQktLi/qzZUuJF5Bdgk5I4wi8oB3d9ofv2QZdrkHOsV8D/3YrkHhNW756A/DHPwWu+HCJakpERKRVcNCk1+vx4IMPIpFIYO/evbj11lvR2toKi8WCkZER7N69G5FIBENDQ6Wsb97mSygPBoNZn7tz506Mjo6qP8ePH8+6L+XGG3wNM/KFx6ubGnC7YXNuTz4SAL7xMeCspC1vvwT41CPAxTtKVEsiIqJ0Bec0KV1vNptNXXNuz5496O3tRUtLS1GVEkUx6+g4JZcpWzCUepxcEsqzaWpqQlNT07znodxMz8gYCmoDz4/uuBirm3K4DZ/7AfC9u4GZSW35pi7g94aAVe2lqygREVEGRQ0tamtrw+7du0sSKM1mMBiydpspQc58I+eUfeabmqCrqyvv+lFhfnXkJF6XxjVlH+/OYZqB8DeBH/15+qK7l94KWL8GLFtVwloSERFlVtQ8TU6nE/39/SUNmIDkbN3Zgp1IJAKTyZTTcfr6+hAOhzNuU46fS4sVlca+g9pWpqs2NuPqTc1Z9j7vSTfwwz9LD5iu6QXu+g8GTEREVDEFB01+vx933313Keui6u3tRSwWyxg4+Xy+tOVPJElCIBBI29disUAQhIxrzHm9XthsNk45UCEjpybwyPNvasruun7L3AngT/w/4KcZRl92fTK56G59Y4lrSURElF3BQVNPT08p66EhCAIGBwfhdDo15T6fD6Iopk0hYLVaYTabMy7k6/V64XQ6Nd19Ho8H0WgULperLPWndN8/9Dompy9kgDc11OF3r9uU/Qm/+hfg4fvSy2/8S+Aj/8g15IiIqOJKMl1yIpFQW4V27NihPi50RnDgQiuR0+mEXq9Xgx6/35+2r9lsRjAYzJifpMwd1d/fj7a2NsRiMYiiiEgkUnDdKD+yLOO7KV1zH75mI1pWZmkp+uU/AYH/nV5+898CH/hs6StIRESUA50sy/L8u2X26quvwul0qt1fOp0OU1NT+NznPof7778fdrsdX/nKV0pW2WpLJBJoaWnB6Oho2sSelF3oaAx3fvUJTdl3be/Be8QMI95+9S+A/2/Ty01fAG78i/JUkIiIFrVSfX8X3NI0OjqqjnJzOByIx+PYu3cvAKhzNLndbpjNZtx+++0FV5AWvu8+pW1l2t6+Eu/uyDA56VODmQOmW74I3PA/y1Q7IiKi3BScGOJ0OjE6OopoNIrdu3fjlltu0Wz3er3o7OzEnj17iq4kLVynJ6bwX795Q1PW1701PQE8/E3gJxm63m79EgMmIiKqCQUHTYFAABaLBdu3b8+6T19f35wzbtPi99/PvonxyWn1cX2dDncaUxLAn/EBP8wQGN3yReC9f1rmGhIREeWm4KApGo3OO8fRyMhIoYenReKhQ9o14m66bB3Wr1l+oeDlh4GHbABSUus+9NdsYSIioppScNBkMBiyThypCAQCnHF7CTshjePxiDZwvsMwq5Xp2JPA0B8B8rT2iTf+JfCBeytQQyIiotwVHDT19vYiEAjg0Ucfzbj9/vvvx6FDh9ImoqSl4weHX8fssZlrmhpgunJD8sHbLwLf7gWmtMuq4N1/AvR8Hphr0ksiIqIqKHj0nMPhwL59+2AymTAwMICZmeQyF08//TT27Nmjjpy74447SlZZWjhkWcb3w69ryj5y7UYsb6wHRl8DvnUHcFbSPunau4BbdzFgIiKimlTU5JYHDhyAxWLBvffeC51OB1mWYTAYIMsyHA4Hdu/eXap60gLz7OsJvPL2KU3ZHYbNwLgEfPMOIKENqHDpLcDv/itn+iYioppVVNDU0tICv9+PQ4cOIRAIYGRkBHq9HiaTCR0dHaWqIy1A3wtrE8A3t65A1+bVwLctwMmXtDtv7gasX+dackREVNNKsoxKZ2cnOjs7S3EoWgQmp2fwo6dPaMru2HEx6n7yV8Dwz7U7r70c+L0hYNmqCtaQiIgof+wLoZJ77OV3MHL6nKbsE/hP4NA3tTuu3gD8gQ9YmWF2cCIiohpT8qBpdHQUl1xyCdrb29He3o7bbrut1KegGvdQSgL4n65/Fm1PfEm7U8MK4OPfBYStFawZERFR4UrSPTdbS0sLOjs71SVW/H5/qU9BNezUxBQCL7ylPr5SdxR/eeofU/bSAXcOApsMla0cERFREcrSPef1evHII4/A4XCU4/BUw/zPv4mJqeT0E61IwLPsH9Ewc1a70y1/D1z5O1WoHRERUeHKmtOUtigrLXo/ejq5OG89pvGvjf8XW3TvaHcw/BHw3j+rQs2IiIiKw0RwKhnpzDk89nIySLqv4dt4X/1z2h22vAf48AOcvJKIiBaknIKmvXv3IpFIlLsutMD99Nk3MTUj42N1v8SnGn6q3bhmI9D7DaBhWXUqR0REVKScgiabzYbBwcFy14UWuB89fQKX6F7Dlxof1G6oXwb0fQtYs6E6FSMiIiqBnEfPHTlyBAcOHMjr4KFQKO8K0cL0duIsDkdfxw8a/xkrdRPajb/9T8DmrupUjIiIqER0sjx7HfrM6urqCkrqlmUZOp0O09PTBVWu1iQSCbS0tGB0dBTNzc3Vrk5N+dqvhrHmv/8clvrHtBuMnwB+55+rUiciIiKgdN/fObc02Ww2WCyWvA4+NDSEvXv35l0pWnhO//rr+OPUgGnDNcBtXLSZiIgWh5yCJkEQYDab0dPTk9fB4/E4vF5vQRWjheONVw7hU4mvALMaI6caVqGh99+BxhXVqxgREVEJ5dQ9R0nsnstgagIn/+l9WHv6FW3x7Q+i4br8WiaJiIjKoVTf35yniYpz4ItpAdMTbR9jwERERIsOgyYq3Ku/hPz4/9UUvTizBTO3fCnLE4iIiBYuBk1UmLOjwPf/BDpc6N2dkBvwt/X/H66/dGMVK0ZERFQeDJqoMD+5Fxg9rim6f6oP26+6Ho31vK2IiGjx4bcb5e+FHwG/2acpenz6Kjw4/Vv48DVsZSIiosWJQRPlZzwO/PgzmqKEvBKfmbwHq5uW4YZL2qtUMSIiovJi0ET5efhvgFNvaYr+dvITeAPtMF21AU0N9VWqGBERUXkxaKLcRQ4Ah7+lKQpMd+I/Z94HALjt6ouqUSsiIqKKYNBEuZk4Bfzo/9MUjckr8DeTnwSgw8pl9bjpsnXVqRsREVEFMGii3Bz4IiAd0xTtnvo43kQyh+lDV6zH8kZ2zRER0eLFoInmd+IQ8OQeTdFTM1fg29M3q48/fDVHzRER0eLGoInmNjMD/PizwKxJLKfrmuCc7Id8/vZpaqjDBy9n1xwRES1uBQdNiUSilPWgWvX0t4HXg5qiHwm/j2H5QsvSTZetw6qmhkrXjIiIqKIKDpo6Ojrw4IMPlrIuVGvG44D/85qimTY9/tc7N2vKbnkXR80REdHiV3DQFI/HMTQ0hKeffrqU9aFa8uiXgDMnNUVPX70TickLt02dDrj5ivWVrhkREVHFFZXT5Pf7YTAY0N7ejvvuuw+HDx8uUbWo6t58Bji4V1t2xW/ju7HLNUVd29vQtmpZBStGRERUHUUFTQMDA3jkkUdgsViwZ88eGI1GtLe349Of/jQeffTRUtWRKk2WgZ84AHnmQlnDckzf8g/Y/6J2NvBbrtpQ4coRERFVR8FBk9frhc1mQ09PD9xuN2KxGA4ePIi7774bjzzyCHp6etDe3o677rqLAdRC89JPgGOPa8ve/xkcHmvGyVPnNMVmBk1ERLREFBw03XnnnWhubtaUGQwGuFwuHDlyBC6XS8176unpQX19Pe666y7mQNW66Skg8L+1ZcJW4IY/xyPPa1uZLtuwGtvaV1WubkRERFVUcNCUKX/pwIEDuOeee1BfX4/Pfe5zMJlM8Hq9CIVC2LVrF4LBIAwGAz796U8XU2cqp0PfBE6+rC3r+TzQuBz+lKCJrUxERLSUFDy5jtFohMfjUbvnPB4P4vE4RFHE7t27YbPZ0NLSou7f2dkJh8MBn8+H3t5eCIKAL33pSyV5EVQi504DP9ulLdu4A3jXHYi8cwrRd05rNpmv4lQDRES0dBQcNMmyDJvNBgBoaWmBzWZDX18fOjs753yexWKBzWaD2+1m0FRrnvh/wCltaxLMfwfU1aW1Mq1f04RrN7WAiIhoqShqGmeDwYDdu3ejp6cn5+f09vbie9/7HkRRLObUVGqn3gF+9c/asktMgHgTAKQFTT1XbkBdna5StSMiIqq6oqYcsNvteQVMQLKFqqOjA263u5hTU6k9dj9w7tSsAh1g+gIA4OSpCYSPxTW7c6oBIiJaagpuaQqFQvN2xWXi9XoLPSWVS+INIPR1bdl1HwcuuhoA8OiLb0O+sF4vVi6rx3v17ZWrHxERUQ0ouKWpkICJatSv/hmYnrjwuK4R+NB96sNHX3pbs/v7L12L5Y31laodERFRTSjplAMAMDw8jPb2dlx//fX4/ve/X+jhqVLG3gJCX9OWdf4BIGwBAExOz+AXL2vXn+Nac0REtBQVHDQZjUY8+OCDaeUdHR24+eabEYvFYLFYOBt4rXv8X4Cpsxce1zUAN/6l+vDgqzGMTUxpnvKhyxk0ERHR0lNw0CTLMuLxeMZtXq8XR44cwY4dO+B0OguuHJXZqXeA4L9py3b8HtC6TX144AVt19zVm5qxvnl5JWpHRERUU4oaPTcfs9mMUChUzlNQMZ74V2DyzIXHunrgxr/S7HIgJZ/p5is4ao6IiJamsgZNkiRBEIRynoIKdXoEeGpQW3bdXUBbh/rw6MjptFnAmc9ERERLVV5TDhw+fBjRaFR9fPDgQTz00ENp+8ViMUQiEQwODsJisRRfSyq9pzzA5KyASFcHvP8zml0OvKhtZVq7ehlnAScioiUrr6Dp5ptvhiRJ6mOv1zvnvEsGgwGDg4NZt1OVTJ4FDu7Vll1jBdr1mqLUoOmmy9ZzFnAiIlqy8gqaQqEQJEmCLMvo6uqC3W5X159LJQgCOjo6Mm6jKntmCDijnUYA7/sLzcPTE1N4MhrTlLFrjoiIlrK8gqbUIEiv15d1kstAIAC/3w+9Xq+2cDkcjqKPOzAwAIPBAJPJVPSxFhxZBp74irZM/CCw4SpN0S+PnMS56Rn1cUOdDu+/bG0FKkhERFSbilqwt5xJ3j6fD/v27dN0/wUCAZjNZvj9/oKPK0kSnE7n0l3OJfoo8M4L2rL3/Gnabo+mdM11b29D8/LGctaMiIiophU8ei4UCuHuu+8uZV1UkiShv78/LR/KZDIhFovB4/EUfOxinrsopLYytV8KXKJtcZNlOW3pFHbNERHRUleTa88NDQ1BFMWMLVl9fX1wu90FHTcQCCzNLjnFOy8DR1Ja6d5zD1CnvQ1efHMMbyUmNGUfYtBERERLXFnnaSqU1+tFW1tbxm2iKCIcDmtG8eUqHA7DYDAUWbsF7Mmvah8vF4DrPp6222Mvv6N5vLl1BfTrVpWxYkRERLWvrEHT/v370d3dnffzgsEgRFHMuE0pnz1fVC48Hk9JksgXrDMx4PB3tGVdfwwsW5m2689TgqabLlsHnY5TDRAR0dJW1qApGo0iHA7n/bxcZhLPJ2iKRqNZW66WjKe/A0yNX3hc1wBcnz5dxOmJKRx8VTvVwE2XrSt37YiIiGpeTXbPzUUJpmKx2Nw7zuLz+Zb2zOSyDIS/oS276neB5ovTdn0iMoLJaVl93FCnw3v17eWuIRERUc3LacoBZfmUO+64Qy176KGHcPDgwTmfVwuL9RYTME1MTGBi4kJCdCKRKFW1Kuu1IPDOi9oy4ycy7vrYK9quOeO2VqzhVANERES5BU0333wzRkdHEYlEsH37dgCAxWKBTqeDLMtzPrfUuTBKAngu3W2SJCEWi2XNj5rPrl278IUvfKGg59aU8L9rH7duB7bdmHHXtHymy9k1R0REBOTYPbd792709/erAZPC4XAgHo9n/dm9e3c56pwzj8eTdZmXXOzcuROjo6Pqz/Hjx0tYuwqZGAOeTVlUufMP06YZAIBXT57G0ZEzmjLmMxERESXl1NKULfBob29HS0v2Ve9bW1sLqpQoilkTvZVcpvlaj0oxvUBTUxOampqKOkbVPfd9YPL0hce6OmDH72fcNbWVad2aJly1sbmctSMiIlowilpGpVwMBkPWeZiUYGq+gCgajWLfvn1pE2Eqx921axf27duHtra2gifLXBBSE8AvvQVo3phx19T5mT5wKacaICIiUhQcNNlstnln1xZFsaCZw81mM1wuV8ZtkUgkp1m9LRZLxgRwSZLQ2tqKnTt3Lv4RdW+/ALyWkqxv+B8Zd52YmsbjkRFN2Qe4QC8REZGq4CkH9uzZgx07dsy5T09PD4LBYN7H7u3tRSwWy9hF5/P5YLfbNWWSJCEQCOR9nkUv/E3t49Ubki1NGQRfjWN8clp9rNMB77+U+UxERESKmpynSRAEDA4Owul0asp9Ph9EUUxrIbJarTCbzTktxqsEYvnM87QgTZ1LTmg523UfB+ozTx+Q2jV37WYBbauWlat2REREC05N5jQBye41QRDgdDqh1+vVXCS/35+2r9lsRjAYRFdXV9bjhcNh7Nq1Sw2anE4n/H4/zGZzUSPsalb0UWA8JTDM0jUHZF46hYiIiC7QyfNNtFSARCKB5ubFN+oqkUigpaUFo6Ojtf/6vv8n2pamzdcDd6cHnADwztgEuv9B273p+5P3omv7El96hoiIFoVSfX8X3D134MABPPDAA3jggQdw+PBhtby7uxutra2or6/HfffdV3DFqAhTE8CLP9aWXX1H5n0B/OrISc3j1U0NuG6LUIaKERERLVxFjZ4bHh6GLMswm814+OGH0dvbi0gkgj179uDIkSNwuVzo7u7G7bffXso603wiB4CJ2Uu+6JJrzWXxi1e0QdN7xHY01tdkuhsREVHVFPzNaDKZIMsyvF4vHn74YYyOjsLn82FgYAD9/f1wuVzo7OzEl770pVLWl3KROgP41vdmXJwXAGRZxi+PaPOZ3n8ppxogIiJKVXDQNDQ0BKvVijvvvBMAEAgEoNPpNHMomUymrDN7U5lMjgMv/URb9q7sLX1H3j6FtxITmrIbGTQRERGlKThokiQJ3d3d6uODBw9CEATN+nTt7e1ZZ/amMjmyHzh3alaBDrjqo1l3/2VKPtPGluUQ164qU+WIiIgWrqISV0ZGLswgHQ6H04b8RyIRCIJQzCkoX899X/t42/uANRdl3f2XKflMN16ylkunEBERZVBw0GQwGBAOh9XHwWBQsx7c8PAwPB5PTkueUImcOwO89FNt2dXZu+Ymp2fw66h26RR2zREREWVWcNBks9kQCATw4IMP4p577sHo6CjsdjsSiQR27tyJSy65BDqdLm3JEyqjI35g8vSFx7o64MrsXXOHjkk4fW5aU/a+Sxg0ERERZVLUlAN+vx/9/f0AgIGBAWzfvh2Dg4OaxXatVqumG4/KKLVrbvuNwOr1WXf/5SvaUXNXbmzG2tVN5agZERHRglfUMiperxfDw8Noa2tDS0sLAKC/v1+T28ScpgqZmgBefkRbNseoOQD4RUoSOKcaICIiyq7otec6OjrSyjo7O4s9LOXr2BParjkAuOJ3su4+Oj6Jp49LmrIb2TVHRESUFad9XixeSVlX7uJOYHX2RXd/HR3BzKxVB5c11OH6Dq41R0RElE3BQdP999+P66+/HolEYv6dqfyOaBfcxSXmOXdPXW+ua1srljfWl7pWREREi0bB3XNOpxM6nQ6xWKyoFYOpBKTjwDsvassuzS9o4qg5IiKiuRXU0jQ6OgogmQg+ewZwqpLUVqYVrcAmY9bd30qcReQdbf4TgyYiIqK5FRQ0KSPlgsHgnPvt379fs9QKlUlq0KS/GajL3tX2REQ7BcSapgZcfTFbC4mIiOZScE7TnXfeCbfbjbGxsaz7RKNRzazhVAZT54Doz7Rl8+QzPR7Rds29W2xHQz3HBBAREc2l4G/KvXv3QpZl9PT04OjRo6WsE+Xj+K9TFugFcElP1t1lWcavjmhbmm7Qt5ejZkRERItKwYngw8PD6O3thcfjgSiKsFqt6OrqgiAIaGtLDl33+/3zHIWKljrVwMYdc84Cfjw2jtelcU3ZDZcwaCIiIppPwUGTwWCATqcDkGy9GBoawtDQUNp+yj5UJqn5TPOMmkvtmmtftQyXb1hT6loREREtOkXNCG6z2WCz2bJu37NnD/bu3VvMKWguo68Bbz+vLZs3n0nbNfdefTsDWyIiohwUFTTp9fo5l0zp6upi0FROqa1MywVgc1fGXYFki2Bq0HSDnlMNEBER5aLgRHBBECCK4pz7tLa2qtMTUBkc2a99PM9UA6+8fQonT01oypgETkRElJuCW5pisdi8+1gsFlgslkJPQXOR5eQivbPpb57zKY+nzAK+SViBbe0rS10zIiKiRamsk/McOHAADzzwQDlPsXTFosDpd7Rl226Y8ynMZyIiIipcWYOmoaEhuN3ucp5i6Tr2a+3jVeuBtuzdpdMzMn4d5fxMREREhSoqEVxx+PDhjN11wWAQgiCU4hSU6nhK0LT1PcAcrUbPnRhF4uyUpoxJ4ERERLkrKmh66KGH0N/fD0mSsu7j9XqLOQVlk9rStPU9c+6eut6cuHYVLmpZXupaERERLVoFd88NDw/DYrGgo6MDe/bsQUtLCywWCx555BEMDQ3h7rvvBoB5R9hRAU6PACdf1pbNEzSlds29l11zREREeSm4pcnpdKK1tRXBYBAA4Ha7IYoienqS657deeed2L9/P3bv3o3vfve7paktJR1/Uvu4cSVw0bVZd5+ansHBV+OasveIDJqIiIjyUXBLUzgc1swG3tbWhmg0qtnHYrEgFAoVXjvKLHWqgU1GoL4x6+7PnUjg1IQ2n+ndYls5akZERLRoFRw0RaNRtLdfaK0wGAwIh8Oafdrb29MCKSqB1JamPLvm9OtWYf0a5jMRERHlo6gpB2aPmDOZTIhGo3j66afVskgkwtFzpTZ5FjhxSFuWZ9DErjkiIqL8FRw0dXZ2wuVyYefOnQCSQdP27dvR39+Po0ePYv/+/RgcHERvb2/JKktIBkzT5y481tUBm6/PujvzmYiIiEqj4KBpcHAQHR0d0Ov1apnb7UYwGIQoirjlllvQ0tICl8tVkorSean5TOvfBSxvzro785mIiIhKo+DRcwaDAUeOHNGUmUwmhEIh7Nu3D+3t7bDZbGhuzv6FTgXIc34m5jMRERGVRklmBJ+ts7MTnZ2dpT4sAcDMTNFJ4OyaIyIiKkxZ156jEjv5EnBW0pbNETQxn4mIiKh0im5pSiQS2LVrF8LhMKLRKAwGA66//np85jOfKUX9aLbUrrmWLUDL5qy7M5+JiIiodIpqaXrggQfQ2toKl8uFgwcPoqWlBX6/H/feey/Wrl2LRx99tFT1JAB447D28Zbso+YA5jMRERGVUsFB0/e+9z04HA709PQgEokgFoshGAwiFovhkUceQUtLC0wmE44ePVrK+i5tbz2nfbzxujl353pzREREpVNw0OR0OmE0GvHII4+go6NDs81kMsHv96OlpQVOp7PoShKSSeBvPa8t2/CurLszn4mIiKi0CgqaRkdHEY1GYbfbs+4jiiJ6e3u59lypSK8Ck6e1ZRuuzrp7xnymDgZNREREhSooaGppaQEA6HS6OfcTBEGz1AoV4c1ntY9XrgVWb8i6+1PD2usurluFdWuaylEzIiKiJaGoZVT27NmTdfvo6Ch8Ph96enrStu3fvx+XXnppoademlLzmTa8C5gjaH0yJWhiKxMREVFxCg6a9u7di1AohLvuuist2fvAgQPo6upCPB7H3r17054bjUYRjUYLPfXS9FZKS9McXXMzMzIOvpoaNHGqASIiomIUPE+T1WoFAAwNDcHr9aZtl2UZANDa2lroKWi2TC1NWbz01hhGxyc1ZdczaCIiIipKwUFTNBqF0WiE0WjM+7nBYBCHDh0q9NRLz8QpID6sLZsjaErNZ9rStgIXCyvKUTMiIqIlo6gZwfv6+vDZz3427+d5PB7cc889xZx6aXn7Be1jXT2w7oqsu6cGTddvZz4TERFRsQrOaXK5XLBYLAU912w2Y/fu3YWeeul56xnt47WXAo2ZZ/aWZTlDEji75oiIiIpVcEvTvffeW/BJOzo6inr+kpNHPlP05GmcPDWhKeN6c0RERMUresHe2R566CFEo1GIogiTyYTm5uZSHn7pyiNoSu2a29DchK1tK8tRKyIioiUlp6DplltuwfCwNhHZ7Xbj5ptvBpCcYsBqtUKSJM2oub179+L2228vcZWXGFnOEDRln24gLZ+po33eSUiJiIhofjnlNFksFkQiEfWnpaUFoigCAIaHh2EymSDLMoaGhhCPxxEMBmEwGGCxWPD000+X9QUseqPHgYmEtiyvoIldc0RERKWQU9Ck1+sBJBO4laBo+/btAJIJ4TqdDgcOHMCdd96JlpYWGAwG+P1+dHR0cMHeYqUun7JcAJovzrjr8dgZvC6Na8rew6CJiIioJHIKmlwuF/R6PR5++GF13TlFIBCAKIrYsWNH2vMsFgsOHjxYkoouWZm65rJ0t6W2MrWtWoZL1q8uV82IiIiWlJxymoLBIPr6+tLKR0dHEY1GYbfbMz6vvb0dkiQVXLlAIAC/3w+9Xq8ex+Fw5H0cj8eDSCSCcDiMWCwGk8kEl8tVcL0qKm35lNyTwLu3tzKfiYiIqERyCpokSYIgCGnlgUAAOp0OZrM54/MOHjyo5j7ly+fzYd++fZolWgKBAMxmM/x+f87HcTqdsNvtsNlsAJKvxWq1orW1FcPDwxlfV03JZ+Rc2npznNSSiIioVHLqnuvs7EQgEEgrd7vdAACTyZS2bXR0FD6fL+O2+UiShP7+fgwODmrKTSYTYrEYPB5PTsfx+Xzo6+vTBG6CIMDr9arBU007dwaIRbRlF2VOAn977CyGT57WlDEJnIiIqHRyCpp27tyJUCiEL3/5y2rZAw88gEAgAIvFkjYf06uvvoqenh7odLqCEsGHhoYgimLGVqC+vj41WJvPwYMHYTAY0soFQYDNZkMgECiq+7Ds3nkRkGdmFeiAdVdm3DX4alzzeHVTA67cyHmyiIiISiXnKQfuvPNO3HvvvWhvb0d7ezscDgdEUdS0Bh06dAjt7e3Q6/UIh8MAkHOr0GxerxdtbZlbSURRRDgczinY8Xg8WbsOlYWGg8Fg3vWrmJEj2sdtHcCyzBNVpuYzGba1or6O+UxERESlkvOM4F6vFz6fD4FAALFYDGazGf39/Wn7GQwGTfJxPB5P22c+wWAQvb29GbcpXW3RaDRjK9JsXV1dWbcpQVe24KwmxI9qH7d2ZN314Kupi/S2lqNGRERES1Zey6hYLJY5F+nt7OzMK0k7m2yJ57PlEjTNVZdIJJkrNN8xqkp6Vfu4dVvG3cbOTuKFN7QTYHZvr+FgkIhKSpZlTE5OYmZmZv6diRaguro6NDY2Vn1EeEnXnqsEJZiKxWJz7zgPj8ejjqirWaktTULmoCl8TMKMfOFxY70O120RylcvIqoJ09PTOHnyJMbGxjA5OVnt6hCVVWNjI9asWYO1a9eivr6+KnVYcEFTKTidToiiWPtzNUmp3XOZg6aDKflM124WsLyxOjcUEVXG9PQ0jh8/jomJCbS0tGD16tWor6+v+l/iRKUmyzKmp6dx6tQpSJKE8fFxbNmypSqB04ILmorNRQqHw/B4PAiFQvN2AU5MTGBiYkJ9nEgk5ti7xKangNHXtWVZWppS85nYNUe0+J08eRITExPYunUrVqxYUe3qEJXd6tWr0dLSgmPHjuHkyZPYsGFDxeuQ0+i5xcRqtWL//v05Tbq5a9cutLS0qD9btmypQA3PS7wOyNPasgxB08TUNA4flzRl3UwCJ1rUZFnG2NgYWlpaGDDRkrJixQo0NzdjbGwMsizP/4QSq8mgSRRFRKPRjNuUXKZCZho3m81wu905J3/v3LkTo6Oj6s/x48fzPmfBUrvmlq0GVqa3ID37+igmpi4kf+p0QNc2tjQRLWaTk5OYnJzE6tVcW5KWnjVr1qjvgUqrye45g8GQdR4mJZjKd9Sb3W6H0+nMa4bypqYmNDU15XWeksmUBJ4hV+FgyqSWl29Yg5aVjeWsGRFVmTJKrlrJsETVpNz31RgtWpMtTWazOWtLUyQSyXtploGBAVit1rTnRaPRjMvD1IQCk8C72DVHtGQw6ZuWomre9zUZNPX29iIWi2UMnHw+H+x2u6ZMkqSswY/P54PBYMgYaIXD4YIXFC476Zj2sbA1bZeZGRnBo9qWJiaBExERlUdNds8JgoDBwUE4nU54vV613OfzQRTFtAk2rVYrAoEA3G63Zu6lcDgMt9sNq9WqWc5F6frbt28fQqFQeV9MoXKYo+mVt09hdFzbp8tFeomIiMqjJoMmIDn7uCAIcDqd0Ov1aqCTaZZvs9mMYDCYtmxKT0/PnK1QNdvKBOTUPfdUylQDm4QV2NjCkTRERETlULNBEwCYTKac8pccDgccDkdaeSHr3tWEybPA2BvasgwtTcHU9ebYykRERFQ2NZnTtOSNZpjaIENOUzBl5ByTwImIiMqHQVMtSu2aW9EKLG/WFJ2QxvG6NK4pu55J4ERERGXDoKkW5ZAEnjpqrmVFI/TrONEdEVGuWltbodPpND+zBw1Vy8DAgFofo9FY1LEkSdK8vnA4XKJaFs5oNNbEdS4Eg6ZalEMSeGo+U9e2VtTVcc4WIqJcDQ4OqiOubTYbXC4Xent7q1yr5EAol8sFg8FQdJAjCIJmZHkwGCxFFQs2MDCAcDiMSCRS1XoUikFTLcqhpSl1JnAj85mIiPJisVhgtVoBJKeucTgc8y7kXgmiKMLhcKSNCC+UzWZLm9+wGjweD5xOZ7WrURQGTbUotaUpJQk8cXYSL72Z0JRxUksiIqpF4XAYOp0Odrs97xU9ak1NTzmwZKXOBt66XfPw0DEJM7MWd15WX4drNrWUv15EtCDMzMiInzlX7WqURevKZUxFWGBEUYTb7UZXVxdEUURr68LtGWHQVGsmTgFnRrRlKd1zoZR8pms2t2B5IxfuJKKk+JlzMH6xRtfVLFLob0xoX12lhdTPC4fD6O/vV/ONRFGEwWCAy+VKmzRZWUvVbrcjEolgaGgIbW1t8Pv9CAQCcLlcAAC32521FSYajcLpdKoTNXd1dak5T9nq53Q6EQwG0dbWBovFArPZXJLXUwhBEDSrdSxk7J6rNaldc0Ba91xqPhPnZyIiqhyr1YpwOAyHwwGXywWLxYJAIAC9Xg+fz5e2LwA4nU4MDQ3BZrMhFovBaDSq3VWCIKj7ZWI0GhEIBGCz2dDb24tgMAij0Zh2LiCZaG00GhEMBtHb2wuTyQSPxzNn0JTP61nq2NJUa1KTwFdfBDQuVx9OTs/g8HFJs0vXNuYzERFVitPpRCQSUVuJAMDlcsFoNKK/v1+zPqrNZoPf70c0GoXX61VbkwYGBmCz2eB2uxEIBGA2mxEOhzO2HomiqFkndfa5lKALuNDCZDKZNEuOKftHo9GiX89Sx5amWjNPEvjzJxIYn5zWlBm3saWJiKhSlOkJUtntdkiSlHGaAEEQ1IBJr9cDgDqSTOkCi8Viac8DklMjpB7L6/VCkiTs2rVLLVeON3uhe2X/uUatFfJ6liq2NNWatCRwbT7TwZR8Jv26VWhbtazctSKiBaR15TKE/mZhj1LKpnVl7XzeBQIB+P1+hMNhxGIxtSUnU/CTafoAJVhqa5u7tyBT65PBYIAgCJqAJhgMalqe5jt/qnxez1LFoKnWzDNHUyhlJnBONUBEqerqdFVPll7MAoEA7HY7otGomjRtMBgQjUYrmgMkiqKmy02SpIISt2vl9SwEDJpqzRyzgcuynD6pJbvmiIjKSpIkTd6Q2WyGwWBAKBTStAIFAoGKBhnRaFQz4k4QhKytQtnKa+n1LATMaaolspyhpelCTtOx2BmcPDWh2cyWJiKi8rFarZr13/bt2wcgmTeU2m0mSVJZ6qBMNZBaJkkSuru71bKuri61PFVqnpOiGq9nIWPQVEvG48C5MW3ZrO651FamtaubsK19ZSVqRkS0JEmSpGmlaW9vB4CMI9Fmj1grpdQlUJR5nwRBgMPhUMtdLhckSUJ/f79m/3A4nHWB3Gq8noWM3XO15PTJ9LI1G9X/ho6mL9Kr03FmXCKiQvh8PjUw8Hq9GQOH1DKbzYZdu3bBbDbDZrNBr9djZGQEPp9P3dfpdKKrq0ude0lJ1g4EAujq6lLP6fF4YLPZ1IDG7/ejq6srLZE7FotBr9fDYrFAkiQMDQ1BkqS0oMZgMMDhcKhzNZlMJjUvScl/8nq9iEQi2LlzpzrpZD6vp9BlUJTXqLReBQIBtcxkMpVkEs2KkClno6OjMgB5dHS0PCc4+oQsf775ws8XN2o2m778M3mb87/Un8HHIuWpBxHVtPHxcfn555+Xx8fHq12VBU0QBBnAvD8mk0nzvHg8LttsNvX5oijKFotFDoVCstfrlQ0GgyyKouxyuWRRFNXjiKIoe71ezbFDoZCmHm63Wz2P2+2WLRaLHI/HZYvFIguCoJ4rEsn++e/1emWTyaSe0+VyyfF4XK2XKIpyKBQq6PUUIh6Pz3l9bTZbXscr5P4v1fe3TpblWauY0VwSiQRaWlowOjqK5ubm0p/gxZ8A3/34hcctW4C/fBYAMHpmEtf93SOa3b//6RvQuZWJ4ERLzdmzZzE8PIyOjg4sX758/icQLSKF3P+l+v5mTlMtGU8Z3bDiQkAUPqbNZ2pqqMO7LuYivURERJXCoKmWnEkJmla2q/9NnZ/pus0CljXw10dERFQp/NatJWdGtI9XXphOIJiSBG7g/ExEREQVxaCplqR1zyWDpsyL9DJoIiIiqiQGTbUkrXsuGTS98EYCZydnNJvY0kRERFRZDJpqybg2b0lpaQqmTGopcpFeIiKiimPQVEuytDSlJoGza46IiKjyGDTVktScppVtkGU5LQm8axvXmyMiIqo0Bk21QpbTR8+taMPr0jjeSmgX6WU+ExERUeUxaKoVE2PAzJS2bGVbWtecsLIR+nWrKlgxIiIiAhg01Y7UrjkAWJEeNBm3cpFeIiKiamDQVCtSk8DrGoCmNWkj54zb2TVHRERUDQyaakVaEng7Tp2bxotvJjTFTAInIiKqDgZNtSK1pWlFGw4fkzAjXyhqrNfh2s1cpJeIiKgaGDTVigxzNKXmM111cQuWN9ZXsFJERESkYNBUK9LWnWtF6BgntSQiIqoVDJpqRUpLk7yiDYdSR84xaCIiIqoaBk21IqWlKSavxtiEdt4mBk1ERETVw6CpVqS0NB0dX6F5vElYgQ3NyytZIyIiIpqFQVOtSFlC5eWxRs1jtjIRERFVF4OmWjGuzV96Jqb91XRxUksiopJqbU2usDD7x+PxVLtaGBgYUOtjNBqLOpYkSZrXFw6HS1TL/ITDYZjNZrS2tqK1tRVGoxEDAwNVqUsxGDTVipTuuRdHtS1Nhq0MmoiISmlwcBA2mw0AYLPZ4HK50NvbW+VaARaLBS6XCwaDoeggRxAEuN1u9XUGg8FSVDEvAwMDMBqNiEajsNlsal2cTif0ej0kSap4nQrVUO0KEICpCWDytKZIwmr1/yuX1eOKi9ZUulZERIuaxWKBIAjweDywWq0wmUzVrhIAQBRFOBwORCKRkrQM2Ww2hMPhqrSihcNhOJ1OWCwWeL1ezbZAIACz2Yyenh6EQqGK160QbGmqBakTWwKIyReCpB1bBDTU81dFREQLy759+wAALpcrbZvJZILL5UI4HK5at2G+2NJUC1KSwGegw+isliYmgRNRXmZm0ifMXSxWtAF1/CNyoYhGowCQtQtOad0LBoMwGAyVqlbBGDTVgpQPt4S8EjOzGgENDJqIKB/jMeB+fbVrUR73RoBVa6tahXA4jP7+frV1RBRFGAwGuFwuiKKo2ddsNiMajcJutyMSiWBoaAhtbW3w+/0IBAJqC4zb7c7aPRiNRuF0OhEIBAAAXV1das5Ttvo5nU4Eg0G0tbXBYrHAbDaX5PXky+VywWw2L4iAKBcM12tBSvdcXF6teWzYwqCJiKhWWK1WhMNhOBwOuFwuWCwWBAIB6PV6+Hy+tH2BZNLz0NAQbDYbYrEYjEYj7HY7TCYTBEFQ98vEaDQiEAjAZrOht7cXwWAQRqMx7VzAhaTrYDCI3t5emEwmeDyeOYOmfF5PvkRRVBO/M1ES07u6uoo6T6WwpakWpLQ0SbiQz3Tp+tVoWdmY+gwiIqoSp9OJSCSiydNxuVwwGo3o7++HxWJRy202G/x+P6LRKLxer9qaNDAwAJvNBrfbrSZEh8PhjC0yoihqEqVnn0sJuoALLUwmkwl+vz9tf6WrrJjXU2pOp1Nt2VoI2NJUC+ZoaWI+ExFRbVGmJ0hlt9shSVLGpGZBENSASa9Pdp06nU4AULvAYrHMeWiDg4Npx/J6vZAkCbt27VLLleOljlITBEHdVqrXUwpmsxmSJMHtdpfl+OXAlqZakDKxZXxWSxPzmYgobyvakrk/i9GKtmrXQBUIBOD3+xEOhxGLxdSWnEzBT6buJyVYamub+zVlaoUxGAwQBEET0ASDQU3L03znT5XP6ymW1WpVc7pqZaqHXDBoqgUpo+fY0kRERamrq3qy9GIWCARgt9sRjUbVriWDwYBoNFp0DlA+RFHUdLlJklRQ4nalX4/ZbEYgEIDD4YDD4Sj58cuJQVMtSOueS7Y0CSsbIa5dVY0aERHReZIkafKGlNFgoVBI0woUCAQqGjRFo1FNK40gCFlbhbKVV/L1SJKEnp4ehMNhzSzlCwlzmmpBWiJ4sqXJuDW5LhIREVWH1WrVrP+mTNbo9XrTus3KtRyIMtVAapkkSeju7lbLurq61PJUqXlOikq9HkmSYDQaEQ6H4ff7F2TABDBoqg1ZEsGZz0REVF2SJGlaadrb2wEg40i02SPWSslut2seK/M+CYKg6d5yuVyQJAn9/f2a/edaQqUSrycajaKjowOxWAyhUCgth2lgYCDryL5aw+65WpDS0qQkgjOfiYiofHw+nxoYeL3ejF/cqWU2mw27du2C2WyGzWaDXq/HyMgIfD6fuq/T6URXV5c695KSrB0IBNDV1aWe0+PxwGazqQGN3+9HV1dXWiJ3LBaDXq+HxWKBJEkYGhqCJElpQY3BYIDD4VDnajKZTGpekpL/5PV6EYlEsHPnTgiCkPfryTdpOxwOqy11NpsNwWBQs2iwJEnqyL4Fkd8kU85GR0dlAPLo6GjpDjo9Jcufb5HlzzerP7d+7iuyuPPH8umJydKdh4gWjfHxcfn555+Xx8fHq12VBU0QBBnAvD8mk0nzvHg8LttsNvX5oijKFotFDoVCstfrlQ0GgyyKouxyuWRRFNXjiKIoe71ezbFDoZCmHm63Wz2P2+2WLRaLHI/HZYvFIguCoJ4rEolkfV1er1c2mUzqOV0ulxyPx9V6iaIoh0Khgl5PvhwOR07X2Ov15nzMQu7/Un1/62RZlisRnC0GiUQCLS0tGB0dRXNzc2kOenoEuF872uHdZ/8V6zd14Ef/88bSnIOIFpWzZ89ieHgYHR0dWL58ebWrQ1RRhdz/pfr+Zk5TtWVYVFPCanbNERER1ZiazmlSJtrS6/VqFn8hfZ6lOk5ZpCSBn5GbMIFlTAInIiKqMTUbNPl8Puzbt08zTFJZnyefjP5SHads0pLAz4+c2ypUoTJERESUTU12zylDJlPX2zGZTIjFYlmHTpbrOGWVYWLLDc1N2CSsqFKFiIiIKJOaDJqGhoYgimLG9XP6+vpyXtyvVMcpqwxLqBi3cVJLIiKiWlOTQZPX6826gKEoigiHwznNVFqq45RVhtnADVuZz0RERFRrajJoCgaDWRcdVMpzmT20VMcpp3OJk5rHcXkNR84RERHVoJoMmmYvjphNLsFOqY5TTqOxtzSPE3Vr8K6LW6pUGyIiIsqmJoOmuShBULYVmyt9nGKdTWlpWtmyHssaFtyvhYiIaNGr2SkHasHExAQmJibUx4lEovQnSUkEX7vuotKfg4iIiIq24Jo0lMTtbAnepTzOrl270NLSov5s2bKlqHOmmpqewfLJUU3Zpk2bSnoOIlq8uAoWLUXVvO8XXNBUSTt37sTo6Kj6c/z48ZIe/6U3E2jBmKZM3Lq1pOcgosWnri750T09PV3lmhBVnnLfK++DSqrJ7jlRFLMmaCs5SNlGxZXyOE1NTWhqapr3PIW68qI1ePvmB/DGm68j/s6bwHgMN1/UUbbzEdHi0NjYiMbGRpw6dQqrV6+udnWIKmpsbEx9D1RaTQZNBoMh6/xJShBkMBgqdpxyqauvx0U3fRLMYiKifOh0OqxZswaSJKGlpQUrVnAFAVoaxsfHkUgkIAhCVSaBrsmgyWw2w+VyZdwWiURgMpkqehwiolqzdu1ajI+P49ixY2hubsaaNWtQX1/P1QRo0ZFlGdPT0xgbG0MikUBTUxPWrl1blbro5BrMJJQkCR0dHQiFQmndZ3q9Hi6XCxaLRbN/MBhMC4LyPc58EokEWlpaMDo6iubm5gJeGRFR6UxPT+PkyZMYGxvD5ORktatDVFaNjY1Ys2YN1q5di/r6+ryeW6rv75psaRIEAYODg3A6nfB6vWq5z+eDKIppgY7VakUgEIDb7YbNZiv4OEREC0l9fT02bNiA9evXY3JyEjMzM9WuElFZ1NXVobGxseotqTUZNAGAxWKBIAhwOp3Q6/VqbpLf70/b12w2IxgMoqurq6jjEBEtRDqdDsuWLat2NYgWvZrsnqtV7J4jIiJaeEr1/c15moiIiIhywKCJiIiIKAcMmoiIiIhywKCJiIiIKAcMmoiIiIhywKCJiIiIKAcMmoiIiIhyULOTW9YiZUqrRCJR5ZoQERFRrpTv7WKnpmTQlIexsTEAwJYtW6pcEyIiIsrX2NgYWlpaCn4+ZwTPw8zMDE6cOIE1a9aUdP2bRCKBLVu24Pjx45xpvMJ47auH1746eN2rh9e+emRZxtjYGC6++GLU1RWemcSWpjzU1dVh8+bNZTt+c3Mz30hVwmtfPbz21cHrXj289tVRTAuTgongRERERDlg0ERERESUAwZNNaCpqQmf//zn0dTUVO2qLDm89tXDa18dvO7Vw2u/8DERnIiIiCgHbGkiIiIiygFHz1VRIBCA3++HXq+HJEkAAIfDUd1KLTIejweRSAThcBixWAwmkwkulyttP6vVira2NtjtdhgMBkiShGAwCLfbjZ07d8JgMFSh9gtTvteS74PSCIfD2LVrF1wuF0RRnHd/3vOFkyQJTqcTgiBk/DxR5HNv832wQMhUFV6vV7ZYLJoyv98vm0ymKtVo8XE4HHIkElEfx+Nx2WQyyYIgyPF4XLOvyWSSAWh+BEGQ/X5/hWu98OVzLfk+KB2v15t23TP9KO8J3vP5czgcssVikV0ulyyKomyz2bLum8+9zffBwsGcpiqQJAkdHR0YHh6GIAiabUajEXa7HTabrTqVWyR8Ph9EUUz7a1mSJLS2tsJkMsHv96vlTqcTZrMZ4XAYACCKIiwWS0XrvFjkei35Pigtp9MJSZKg1+szbh8ZGQEAtWWE93xxjEYjurq64Ha707blc2/zfbCwsHuuCoaGhiCKYtobBAD6+vrgdrv5JinSwYMHM34BCIIAm80Gj8cDSZI0vwOTyQSTyVTBWi5euVxLvg9KL9MXuMLpdKZ1JfGeL4987m2+DxYWJoJXgdfrRVtbW8ZtoigiHA6rfdpUGI/HA7PZnHGb0WgEAASDwUpWiVLwfVBa3d3dWbd5PB7Y7fYK1mZpy+fe5vtgYWHQVAXBYDBroqZSHo1GK1mlRaerqyvrNuUDKNsHFVUG3wella1rTbmGuSSHU2nkc2/zfbCwsHuuClK7hTKJRqMcvVKE2flKqSKRCACkXd9oNIpAIKDZb+fOnfP+rihdLteS74PKcLlcWbvteM+XRz73Nt8HCwuDphqjvHlisVh1K7KIeTyetByBaDSKcDisKQ+HwzAajQiFQvwSyUMpriXfB6UxV7cc7/nqyOfe5vug9rB7jpYUp9MJURTTEmK9Xm9a94bBYIDBYEB/f38lq7jg8VrWDq/Xm7WFgr8novwxaKoxzLcpn3A4DI/HA7/fn/Nf0WazGT6fr7wVWyLyuZZ8HxTP4/EU1KXDe7688rm3+T6oPQyaaMmwWq3Yv39/XgmxyoeVMpcNFY7XsrLcbnfWEaRz4e+JKDsGTVUgimLW0RBK3zVHupSW2WyG2+3O+Je33W6H0+nM+DzmFOQnn2vJ90H5KPlK2a4f7/nyyufe5vtgYWHQVAXKiIlMlDcPR0qUjvIFkW0Sv6GhoawfWkr5XFMY0AX5XEu+D8pHGRGX7cuW93x55XNv832wsDBoqgKz2Zz1AysSiXCG3hIaGBiA1WpNu6azh1rbbDZ4vd6Mz/f7/Vln66V0+VxLvg/KJxQKzbmd93x55XNv832wsDBoqoLe3l7EYrGMbxSfz8eZe0vE5/PBYDBk/NCZ3XXR3d2dMX9DkiT4fL45VzEnrXyuJd8H5RONRucMenjPl1c+9zbfBwtMtVcMXqoyrWrt9Xq5qnWJhEIh2WQyyW63W/Pjcrlkl8slGwwGzf42m00OhUKaMoPBMOcq5pRZPteS74PyEEVRFgRhzn14zxdHFMW0e3e2fO5tvg8WDp0sy3K1A7elKhAIwO/3Q6/Xq33aDoejupVaJFpbW+dcr0kURXVmcMXAwABGRkYgSRJisRj6+vq46nuB8rmWfB+UntVqhSRJc86MD/Cez9fAwAAOHjyoJtoDyUWPBUHIeO3yubf5PlgYGDQRERER5YA5TUREREQ5YNBERERElAMGTUREREQ5YNBERERElAMGTUREREQ5YNBERERElAMGTUREREQ5YNBERERElAMGTUREREQ5YNBERBUVjUah0+ly+nE6ndWu7rwGBgbU+hqNxmpXh4jKqKHaFSCipUUURbjdbrjdboTDYbhcroz7OZ1OdX2vWqasN7Zv374FUV8iKhzXniOiqrDb7fB4PMj2EdTa2oqurq55F52tFfO9HiJa+NjSREQ1qbe3F3q9vtrVICJSMWgioprkdrtz2k+SJAiCUN7KEBGBieBEVEPC4XDGvKDZydZ6vR7hcBhGoxE6nQ6tra3Q6/Xw+XxzHtvpdKrPMZvN8Hg8OdXHarWitbVVk+wdCASyPkeSJNjtdvU5ZrMZ0Wg06/4ej0etl/J67HY79Hr9nOchospjSxMR1Yxdu3YhEAggHo9rylOTrY1GIwwGg9oa5Xa7YbVa4XK54HA4NM+VJAk9PT0Ih8OwWCwwmUwIh8Ow2+3w+/3wer0Z6+LxeGC32yEIAkwmE7q7uzEyMoKBgQGYzeasuUsdHR0AAJvNph7HaDSmvSYAsFqt8Pl8MJlMcDgcaG9vx8GDB9WALhwOw2Qy5Xr5iKjcZCKiKrDZbDKAtB9BEOZ9jsViSdtmMplkAHI8HteUWywWGYAcCoU05W63WwYgu93utGOFQiEZgGwymdK2uVwu2eFwZK2bKIqaOijn8fv9mv0jkUjW83u9XhmA7HK50rYRUfWwe46Iqsrlcqk/oijm/JxUSqvTrl271LJoNAqfzweHwwGDwaDZ32azwWQyZZwLSinLNHLP4XBknSZBqcfsHKuuri61LrO1tbUBAEKhUNoxlBYxtjIR1RZ2zxFRVaV2p0UikXmfkym4EkURgiBocqKUnKC+vr6Mx7FarQgEAohGo5pjBgIBtXstX0qQNB9BEOByueB0OjE0NISuri4YDAZ0d3fDZDItmKkWiJYStjQRUc1wOBw5j5rLRBRFTYuOEoBla8FSWntmP0f5f6HTHeQzks/hcCASiWjyn5TEc6vVWtD5iah8GDQRUc2SJCmv/aPRqKYbTgl8so1eU8pnB1XK/w8ePJjXuQvh8XjQ1tYGl8sFv9+PeDyOSCQCt9uNQCDAZVmIagyDJiKqSYFAAK2trRmnIMg0FD8QCECSJHR3d6tlSk5Qttarffv2AUhviTIYDOrxMslluoL5KCP4ZudgKXWx2WxwuVwIh8N5B45EVD4MmoioJinBQiwWS9tmt9s1wUQ0GlWnB5idIyWKIiwWCzweT1qgNTAwkHXtu8HBQUiSBKvVmha0OJ1O2O32kq0zNzAwkLElTOla5MSdRLWDieBEVHEejwfBYFD9fyZzJULHYjEYjUZYLBZIkoShoSFIkpTxOYODgwiHwzCbzbBYLBBFEeFwGIFAABaLJS0RHUi2NClJ2h0dHeqSLn6/X00SV7oBo9EoAoGA5vWIogiTyQSPx6MGP0rdMiWY6/V62Gw26PV6jIyMIBAIIBwOZ6wbEVVRtec8IKKlRZmfKJcfQRDkSCSiPleZCykej8sWi0UWBEEWRVG2WCya/TJxOByywWCQAcgGgyHj/EipQqGQbLFYZFEUZUEQZJPJlPY8l8uVVm9lrqZMr0mZLyoSiciCIMh+v199LcprNhgMstfrLeDqElE56WSZS3IT0cJgt9vh8XiyzsZNRFROzGkiIiIiygGDJiIiIqIcMGgiIiIiygGDJiKqedFoNG3EXaa5moiIyomJ4ERU8wYGBtIW1hVFMad16oiISoVBExEREVEO2D1HRERElAMGTUREREQ5YNBERERElAMGTUREREQ5YNBERERElAMGTUREREQ5YNBERERElAMGTUREREQ5YNBERERElIP/H6jmcRK5Qq4xAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "epochs = np.arange(120)\n",
    "target_sparsity_level = 0.9\n",
    "sl = [map(lmbda, epochs) for lmbda in sl_lambdas]\n",
    "\n",
    "plt.plot(epochs, target_sparsity_level * np.array(list(sl[0])), label='Lambda 1')\n",
    "plt.plot(epochs, target_sparsity_level * np.array(list(sl[1])), label='Lambda 2')\n",
    "\n",
    "plt.legend()\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Sparsity Level')\n",
    "plt.title(f'Sparsity level Scheduling (target = {target_sparsity_level})')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7bc3c949",
   "metadata": {},
   "source": [
    "## Prepare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "493cdf38",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zafar/Git/pytorch-dev/pytorch/torch/quantization/observer.py:134: UserWarning: Please use quant_min and quant_max to specify the range for observers.                     reduce_range will be deprecated in a future release of PyTorch.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Model(\n",
       "  (quant): QuantStub(\n",
       "    (activation_post_process): HistogramObserver()\n",
       "  )\n",
       "  (seq): Sequential(\n",
       "    (0): ParametrizedLinear(\n",
       "      in_features=16, out_features=16, bias=True\n",
       "      (activation_post_process): HistogramObserver()\n",
       "      (parametrizations): ModuleDict(\n",
       "        (weight): ParametrizationList(\n",
       "          (0): MulBy()\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (linear): ParametrizedLinear(\n",
       "    in_features=16, out_features=16, bias=True\n",
       "    (activation_post_process): HistogramObserver()\n",
       "    (parametrizations): ModuleDict(\n",
       "      (weight): ParametrizationList(\n",
       "        (0): MulBy()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (dequant): DeQuantStub()\n",
       ")"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Order matters, because the quantization mappings are not aware of the `ParametrizedLinear` =/= nn.Linear\n",
    "tq.prepare(model, inplace=True)\n",
    "sparsifier.prepare()\n",
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c29b44e0",
   "metadata": {},
   "source": [
    "## Step (or set some level of sparsity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2e13dfa4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sparsity before step: 0.00%\n",
      "Target sparsity level: 0.00%\n",
      "Sparsity after step: 21.88%\n",
      "Target sparsity level: 23.11%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zafar/Git/pytorch-dev/pytorch/torch/_tensor.py:557: UserWarning: floor_divide is deprecated, and will be removed in a future version of pytorch. It currently rounds toward 0 (like the 'trunc' function NOT 'floor'). This results in incorrect rounding for negative values.\n",
      "To keep the current behavior, use torch.div(a, b, rounding_mode='trunc'), or for actual floor division, use torch.div(a, b, rounding_mode='floor'). (Triggered internally at  /home/zafar/Git/pytorch-dev/pytorch/aten/src/ATen/native/BinaryOps.cpp:461.)\n",
      "  return torch.floor_divide(self, other)\n"
     ]
    }
   ],
   "source": [
    "print(f'Sparsity before step: {1.0 - model.linear.mask.mean():.2%}')\n",
    "print(f'Target sparsity level: {sparsifier.module_groups[0][\"sparsity_level\"]:.2%}')\n",
    "\n",
    "# This is just to show the change in the scheduler\n",
    "for epoch in range(10):\n",
    "    # ... Training here\n",
    "    sparsifier.step()\n",
    "    scheduler.step()\n",
    "\n",
    "print(f'Sparsity after step: {1.0 - model.linear.mask.mean():.2%}')\n",
    "print(f'Target sparsity level: {sparsifier.module_groups[0][\"sparsity_level\"]:.2%}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fdfd8840",
   "metadata": {},
   "source": [
    "## Calibrate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "64bb1ff9",
   "metadata": {},
   "outputs": [],
   "source": [
    "model(torch.randn(16, 16));"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b14a8432",
   "metadata": {},
   "source": [
    "## Convert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "046628d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== BEFORE CONVERT ===\n",
      "Sparsity level (in the weight): 0.0\n",
      "Sparsity level (in the weight): 0.0\n",
      "Layer seq.0 has parametrizations: True\n",
      "Layer linear has parametrizations: True\n",
      "Layer seq.0 has mask: True\n",
      "Layer linear has mask: True\n"
     ]
    }
   ],
   "source": [
    "print('=== BEFORE CONVERT ===')\n",
    "\n",
    "print(f'Sparsity level (in the weight): {(model.seq[0].parametrizations.weight.original == 0).float().mean()}')\n",
    "print(f'Sparsity level (in the weight): {(model.linear.parametrizations.weight.original == 0).float().mean()}')\n",
    "\n",
    "print(f'Layer seq.0 has parametrizations: {hasattr(model.seq[0], \"parametrizations\")}')\n",
    "print(f'Layer linear has parametrizations: {hasattr(model.linear, \"parametrizations\")}')\n",
    "print(f'Layer seq.0 has mask: {hasattr(model.seq[0], \"mask\")}')\n",
    "print(f'Layer linear has mask: {hasattr(model.linear, \"mask\")}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "572301ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.ao.nn.sparse.quantized as ao_qnn\n",
    "from torch.ao.nn.sparse.quantized.utils import QNNPACKLinearBlockSparsePattern\n",
    "\n",
    "# Order matters, as we want to fuse the mask BEFORE converting the layers into quantized\n",
    "sparsifier.convert()\n",
    "\n",
    "# We need to tell that the mappings are different now\n",
    "sparse_mapping = tq.get_default_static_quant_module_mappings()\n",
    "sparse_mapping[nn.Linear] = ao_qnn.Linear  # Also could be ao_nn.quantized.dynamic.Linear\n",
    "\n",
    "# This is a stopgap measure until we figure out a better way\n",
    "with QNNPACKLinearBlockSparsePattern(1, 4):\n",
    "    tq.convert(model, inplace=True, mapping=sparse_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b9dfd31f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== AFTER CONVERT ===\n",
      "Sparsity level (in the weight): 0.25390625\n",
      "Sparsity level (in the weight): 0.22265625\n",
      "Layer seq.0 has parametrizations: False\n",
      "Layer linear has parametrizations: False\n",
      "Layer seq.0 has mask: False\n",
      "Layer linear has mask: False\n"
     ]
    }
   ],
   "source": [
    "print('=== AFTER CONVERT ===')\n",
    "\n",
    "print(f'Sparsity level (in the weight): {(model.seq[0].weight() == 0).float().mean()}')\n",
    "print(f'Sparsity level (in the weight): {(model.linear.weight() == 0).float().mean()}')\n",
    "\n",
    "print(f'Layer seq.0 has parametrizations: {hasattr(model.seq[0], \"parametrizations\")}')\n",
    "print(f'Layer linear has parametrizations: {hasattr(model.linear, \"parametrizations\")}')\n",
    "print(f'Layer seq.0 has mask: {hasattr(model.seq[0], \"mask\")}')\n",
    "print(f'Layer linear has mask: {hasattr(model.linear, \"mask\")}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}

PTS or SAT Flow.ipynb

SAT + QAT Flow.ipynb