paulaksm/download.sh

## download.sh
#!/bin/bash
wget https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-white.csv

## testing.ipynb
{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import rl_functions\n",
    "import util\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import plots\n",
    "import pandas as pd\n",
    "import subprocess\n",
    "import os\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Download Wine Quality dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(\"winequality-white.csv\"):\n",
    "    pro = subprocess.Popen([\"bash\", \"download.sh\"])\n",
    "    pro.wait()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data and transform to ndarray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fixed acidity</th>\n",
       "      <th>volatile acidity</th>\n",
       "      <th>citric acid</th>\n",
       "      <th>residual sugar</th>\n",
       "      <th>chlorides</th>\n",
       "      <th>free sulfur dioxide</th>\n",
       "      <th>total sulfur dioxide</th>\n",
       "      <th>density</th>\n",
       "      <th>pH</th>\n",
       "      <th>sulphates</th>\n",
       "      <th>alcohol</th>\n",
       "      <th>quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7.0</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.36</td>\n",
       "      <td>20.7</td>\n",
       "      <td>0.045</td>\n",
       "      <td>45.0</td>\n",
       "      <td>170.0</td>\n",
       "      <td>1.0010</td>\n",
       "      <td>3.00</td>\n",
       "      <td>0.45</td>\n",
       "      <td>8.8</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6.3</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.34</td>\n",
       "      <td>1.6</td>\n",
       "      <td>0.049</td>\n",
       "      <td>14.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>0.9940</td>\n",
       "      <td>3.30</td>\n",
       "      <td>0.49</td>\n",
       "      <td>9.5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8.1</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.40</td>\n",
       "      <td>6.9</td>\n",
       "      <td>0.050</td>\n",
       "      <td>30.0</td>\n",
       "      <td>97.0</td>\n",
       "      <td>0.9951</td>\n",
       "      <td>3.26</td>\n",
       "      <td>0.44</td>\n",
       "      <td>10.1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7.2</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.32</td>\n",
       "      <td>8.5</td>\n",
       "      <td>0.058</td>\n",
       "      <td>47.0</td>\n",
       "      <td>186.0</td>\n",
       "      <td>0.9956</td>\n",
       "      <td>3.19</td>\n",
       "      <td>0.40</td>\n",
       "      <td>9.9</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7.2</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.32</td>\n",
       "      <td>8.5</td>\n",
       "      <td>0.058</td>\n",
       "      <td>47.0</td>\n",
       "      <td>186.0</td>\n",
       "      <td>0.9956</td>\n",
       "      <td>3.19</td>\n",
       "      <td>0.40</td>\n",
       "      <td>9.9</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  \\\n",
       "0            7.0              0.27         0.36            20.7      0.045   \n",
       "1            6.3              0.30         0.34             1.6      0.049   \n",
       "2            8.1              0.28         0.40             6.9      0.050   \n",
       "3            7.2              0.23         0.32             8.5      0.058   \n",
       "4            7.2              0.23         0.32             8.5      0.058   \n",
       "\n",
       "   free sulfur dioxide  total sulfur dioxide  density    pH  sulphates  \\\n",
       "0                 45.0                 170.0   1.0010  3.00       0.45   \n",
       "1                 14.0                 132.0   0.9940  3.30       0.49   \n",
       "2                 30.0                  97.0   0.9951  3.26       0.44   \n",
       "3                 47.0                 186.0   0.9956  3.19       0.40   \n",
       "4                 47.0                 186.0   0.9956  3.19       0.40   \n",
       "\n",
       "   alcohol  quality  \n",
       "0      8.8        6  \n",
       "1      9.5        6  \n",
       "2     10.1        6  \n",
       "3      9.9        6  \n",
       "4      9.9        6  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('winequality-white.csv', sep=';')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4898, 12)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = df.values\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Separate target class from features and split in train, test and validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input data shape (4898, 11) Target shape (4898,)\n",
      "\n",
      " Train X shape (4000, 11) \n",
      " Train y shape (4000,) \n",
      " Valid X shape (400, 11) \n",
      " Valid y shape (400,)  \n",
      " Test X shape (498, 11)       \n",
      " Test y shape (498,)\n"
     ]
    }
   ],
   "source": [
    "X = data[:, 0:11]\n",
    "y = data[:, 11]\n",
    "print('Input data shape {} Target shape {}'.format(X.shape, y.shape))\n",
    "train_X = X[0:4000]\n",
    "train_y = y[0:4000]\n",
    "valid_X = X[4000:4400]\n",
    "valid_y = y[4000:4400]\n",
    "test_X = X[4400:4899]\n",
    "test_y = y[4400:4899]\n",
    "print('\\n Train X shape {} \\n Train y shape {} \\n Valid X shape {} \\n Valid y shape {}  \\n Test X shape {} \\\n",
    "      \\n Test y shape {}'.format(train_X.shape,\n",
    "                                 train_y.shape,\n",
    "                                 valid_X.shape,\n",
    "                                 valid_y.shape,\n",
    "                                 test_X.shape,\n",
    "                                 test_y.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Standardize and add bias column "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = rl_functions.standardize(train_X)\n",
    "valid_X = rl_functions.standardize(valid_X)\n",
    "test_X = rl_functions.standardize(test_X)\n",
    "train_X1 = util.add_feature_ones(train_X)\n",
    "valid_X1 = util.add_feature_ones(valid_X)\n",
    "test_X1 = util.add_feature_ones(test_X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Reshape *_y to (N, 1) format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y = np.reshape(train_y, (train_y.shape[0], 1))\n",
    "valid_y = np.reshape(valid_y, (valid_y.shape[0], 1))\n",
    "test_y = np.reshape(test_y, (test_y.shape[0], 1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing batch_gradient_descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAToAAAEoCAYAAADIXD3MAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8XNV99/HPTxpJljdsbCIMTrATTFhiNjvElKaRIVBCyFZoliYtpDwh3eGV5ElDmychTdomTZql2SnkgaQJzkpZQqFArGwFg81iDMbBNpuxHdtg2ZZt7b/+cc5Y12ONNJJGM6N7v+/Xa14z99ztnJk7vznn3HvPmLsjIpJmddXOgIjIeFOgE5HUU6ATkdRToBOR1FOgE5HUU6ATkdRToJNBmdkmM3Mzu3aE6x0b13Mze8945W+Q/X4q7rO3UvuUiUOBbgIxs6cTQaTY4+oy7e5BYAWwYYTrdcb1VgDby5QXkTHJVTsDMiIPAVvj67nA0fH1w0BXfL2p2Mpm1uju3aXsyN3fPJoMuvsmYMlo1hUZL6rRTSDu/jZ3X+LuS4Bkk/JAurtfa2avT9TwLjOzn5tZJ3C5mc03szti03R/fDxqZn+T3Fdh07WgSXqlmX3PzPbE5a5KrHdI09XM/k8i7U1m9su437Vm9oaC/V5kZk+aWWfM94VjaQrH8j8Y97fXzO41s4sKlvmgmT0e5+82szXJJruZLTGzu81sR8zXs2b2UzM7baT5kepQoEu/rwEnAk8BDrQA58XXa4HdwKuAL5nZ+0vc5r8AryPUIo8G/snMlpa47o9jHgCOB5aZ2QwAMzsV+AFwLNANHAksK3G7h4jN+GuB04BthLIuAX5kZpfFZd4GfA44AXiaUCOeD+SDdD1wO3AO0As8DjQBFwCvHG3epLIU6NLvl8Bcdz8B+AbwG2C+u7/U3U8HjgL+Jy77zhK3uQKYRwig+c7/c0pc9wvufhzw7jg9HVgcX3+YcEx2ACe4+yuBr5e43YOY2TTgI3HyJkLwOoaBsn7KzAw4Lk7f6e4nufuJwAzCjwHAbGBmfH2au5/u7i2EIJffltQ4Bbr0+4a7dwG4ex+hpnSVmT1jZj2EQPU7cdmjStzm9929x923AS/EtJahVkj4Tnx+PJGWX/dV8fmX7v58fH1jidsttJBQ8wJY5u79sX/yxzHtSEI/5x1AD/D7ZrbdzH4NfJHwPuHuvwXuj+tsjM3aHwCvBbaMMm9SYQp06ffbgukvA+8HXgZsJNTOdsR59SVusz3xOl+jsxGum7wMpHDdig2p4+6PACcBVwO/IgT7vwB+meiDawXeR2hGdwB/QGgSf7pS+ZSxUaBLv8KgkT8jentsGi5l4ExutT0an3/XzI6Mr981hm3lz0S/w8zqzKwRyJ+I2ApsMrPjgD53/4S7v43Qb7iXcEXC78U+ujOB/+/u740ngq6P2zh7lHmTCtPlJdmzmvBlfoOZrQMOp3Z+8D5L6CecDqwzs62E5uWIufseM/s08HFCDewpoAGYExf5qLu7mZ0NfN3MNhNqv0cCU+Iyq+M69wB7zOw5oJ9QA8zPlwmgVg5wqZwrgFsJtZaphObXf1U1R5G7Pwy8HVhP6F/bRmgy5u0f4faujus/BLyEcJLhPuBid78uLraKcLKim3DmdQrhYun3uvtyQv/dNwnN/KOABcCzhLPZfz3SMkp1mEYYllpiZse5+28S058APhYnF7j7+urkTCYyNV2l1qwys/XAM4QTJvkTAtcpyMloqUYnNcXMvk04y9lCaDY+Qej8/3q8PEZkxBToRCT1dDJCRFJv2ECXGBro+grkZ8TMrDVx03drtfMj4ysej9cPs8z18Xh4ujK5Ghsza4v5bat2XtIqDTW63QyMf7a7ynkZkXEYQ64szGxRHOFkt5ntM7Nfm9m5Ja77ejP7VVxvt5ndaWaLCpb5NzN7Io5+sj8Gr+vM7JgyFWED4Xh4KLHPqgY/M5uX+LwvLZj9OCG/jx+6ZjqY2Rvt4HETpxbML+W4aTCzj5vZRjPrtjByzhfjfc1Dc/chH4QRHRy4frhly/UAGiu1rzLnO0fs9yxxeY+Pq6ud90SeTiZcY+eEgTM3xde9wHnDrPv7cTmP622Pr/cCCxPLbSScVV0FPJl4H54oIX9Pj+ZYJJzQcODpMr5XJR+nhEEQ8uW8tNqfc4WPqRbCxdieeEwdxXHznZjeRzhJ1R2n24C6IfNQ4oF1UKADpgGfJ1xt3k24ufkbwIzEMucTRs7YFpfZHaffUOTD/zDwn8A+wk3VrckDA7gtznsKuCyxjeRyrTHt6kTaUsIFoPvj85KC8r2fcAHovriP9xRub7gvTszfU4Sr5mcAf0y4EXwH4czhTuBO4IxB8px8PJ3Y/nnAz+L7lh+1900VOChviXl5Kn7OOcJFtg6sHmbd1XG5e+N60xgYHuqWxHKTCtb7TuI9mFXC8Xj9MMsc+GwKjuHCR/54OZJw7+rzhGP1GcKF1E2JbbYx8KX6W2AzsDPO+xBh8NMX4+e9HfgJcFycf2mR/bcVbjuxv2bgHwkXT3fHbd8KnJ5YJrndtwC/IBznTwAXJpabAnyVcJx3EgZiWAF8oEKB7nbC7Xj/mchvMtANe9wApyfW/auY9qZE2h+UNdABjYRfYo+Zf4Rwo7PH9IbEh99NaEY8COyJy/QApwwS6LqAXYR7FD/HwcGgOxZ8FwMR/fgSA11n/OB7GAhMubjcBYnlXiDUNDoKtzfMl6k75mcd4f7JGcBX4gG3jvAF6IzL7iZ8qU5nIHjkf8XuA26K276YEDQdeI6BWk8/4ar+Uj6vYo+nh1g3Rwj4Dnwzkf53ifWPKrLu0YllrkqkXxPT9gH1ifQrCV+2ZI3uMYapETO6QHcTA7WErvhe3xc/h1mJ96yDcDx3xelbBwl0XYTax2PAxjjvtrju44TjN187eQ6YBLyR0IzOl3ND3P/Xhgh0dyWWz48bmH8fTx0k0HUThuDKf367gcPjcv+ayPuDcf89wN3DvI9tDH0seQnx46/jsh/g4O/l1JEcN8DfJ5abE5epI3zPHLim3IHuTxgIWCfFtGMSH+67E0EsWcObmfiwPjlIoFubXz4WrDUx74eEES5OTqT9WYmB7q9j2t8k0vJB8hdx+tnEvr9XuL1hvkwOvD+mWXwcB0xOLHtsYtlkbTSfdnXBtjfG9O8ycAnQv8e03wzzed3EwBd5sMdNQ6x7ZCJPn0ykX5ZIX1Jk3SVFyvjJRPqRifQvJtIdWEmRIDrWQFcsLaZ/jIEfuvwX6KxEvs4a5Et/fv44jc8nEX/g4/TrE8ueM8ixfmmRgNIWp5cmlv1Q4rPZGdN+PEig+9eY9uZB8nlrnP5/iX1OB149zPv4tWGOpfuGWf8kQiC6k/C9uDqRt3ygK+m4IbQY89PJH8znYtodQ+VlNHdGvCY+54A1YezCgywhfEEbgevN7HcIv5rJEx+DjXt2g7u3A7h7X8F2v+vubmaDjWE2nGLjnz3BwPhnd+T3TRiKZyQjZuwnBKHw8wbEEXO/GjtTZ3DwMERDjvlmZkcQBokE+CPgjwreiwVmNsvdXzhk5ZCHt40g76UqdQimktd19yvN7IOEe0e/QRix+D/M7Fyv7IXB+eP5cGBzkeP514npde5+B4TjNKa9DPimmZ1MuH+45M+7iFcnXn8v7murmS0H3sbAQKVJQ43zdytwIfAPZvY+Qs3vPg4ejv8Q7v4XI8/6Qb5HaMldEr+/I1m31IVLWm4st4D1EKrBhfLjn/2UUJPpJVTnOwm38zQy+LhnheOmJeUDYG/izSqpgIkANl7jn2139/4DGw5nk+4kBLhOQpOlh4EvVKljvkForm8bJL2h2ApmdhMDI3QMZssQwXAHIXA3E26Cz0u+frbIus8VWT7/ej8F/woWA8UTZvZ5QqBbShip+L+HyP946SA0Rwu1F0wfdJya2csJfU+NhC/1KsL36tS4yEg+77Eoepy7+zVm9gShtrcQWER4n98b7y3eO9gGzexrhOZ9UR6GrSrm5Jif9fF725iYt9XMPgzcnEgb6rgpPL62mFkdoRIFxY9LYHSB7oHEule6+30AZpYDzgXWmtksQpAD+Ji7/7OZzSPUoooZS7AZrUeB3wPOM7Np7r6H0ocTzyvM9ysJQQ7gT939RjNbQuhoLZQPKvlhgXD37fESiHnAGuAid+8BMLOXEYbzHmr8uNMIXQnFPFO0IOGH5B7Cr/958bT9fsIXBOBRd98c8/Jt4Azgfnf/E3d/3szWEGrJbzazz8ay5S9LuTvW1BcSR/aNv/J1hL7SvCmMj33xebKZWb72TTie83217/F4P62ZNRP61u4u2E7h553/8Qb4fXe/18zeyaEjI+9LvB6ujA8kXv8R8Lk4Pl/+fzlWDrP+QczsDOAxd/9FnM4fj0cRhuxaVWTVExn4gR6tHIPHmSmEs9alHjd3AJ+KaRcR+sHfSOgDhTBSdHEl9okk++iaGOhY7Sf8Cq5l4JKEVsIvSb7t3E0IKC8y0NGf39Y8ivdbtCbmtSbSD+rXGmw5En0BQ22Pg09G7CD0je0dbL+l9APF9JmJcu4jnFHaWpjvuOyDMa2LcHD/U0x/R0G+HiKc5esn0WE9Hg/gFAY6tJOXl/QR+3x8kH6lmPaGuJxz8GUC+xg4AfXWmLab0PGffG+eBaaVcDyOpo8u2Ue7jtB0ayb8J8SzDByrqwknSPInkOYVK29MP56B/un8ybTtiX1dmu/RiJ+lE2p+KxjoPx7svSw8GZE/EbefwU9G5PM5b5B9/wehVfEUIajlt9VBoh99vB8M0kdX6nETl8v3n/fF9yR/eckvGObykhFfMOzh/wdaCZeXPE3oYzmCEPA+BazxkKuLCF/ePkL1/d0MDNldE9z9duDPCEF5CuEL8KHEIiMa/yxucyfwh4S+kjrCh/GmIov/DQOj6i4m/lGLu3+f8OH/jFBbOIHwxfsh4Yz0uPEwtPjrCF+0SYSmwb3ABR77poZY978IPx7/E9ebRKgRvS5uF0IQuZUQ6I4n/DCsJ/wJzpkeatXj4VuE/4vYRXifX0Po1N5B6Ie7ltBNcAKho/4BwtnmobpUcPcngD8lBJFGwjF+SB9v/E68j1DWZkJteKia95uBfyL8+L6C8CN3G+HkyMOlFDjhp8DPCZWUhcQzroRLvQqb5hVX4nEDcAnwD4QfplcQ3usvEy6l6WcImb6p38waCP+Q9VQi7TrCgdsNHOHuE+pui7SLzfo2d7+0ylmRCSTr49FNIXSUriI0DY8j/KID/LOCnEg6ZD3QdRKaA68mnCXrJFxK8E13/85QK4rIxJHppquIZEMaRi8RERmSAp2IpN6E7aObPXu2z5s3b8hl9u7dy5Qp43X9aWWpLLUpLWUZSTlWrVq1w92PGOcsldWEDXTz5s1j5cqhLxBva2ujtbW1MhkaZypLbUpLWUZSDjMrendNrVLTVURST4FORFJPgU5EUk+BTkRST4FORFJPgU5EUk+BTkRSL9WB7u5nerj54eernQ0RqbJUB7qfb+rlttVbqp0NEamyVAe6eoPeviEHHhWRDEh1oMvVQU+fhqESybpUB7p6gx7V6EQyL92Brg56+1WjE8m6dAc6M/XRiUjaA5366EQk7YGuDnr7VaMTybpUB7qcanQiQsoDXX2d6ayriKQ80Bn0qkYnknnpDnTqoxMR0h7o1EcnIqQ80OV0r6uIkPJAF05GqEYnknUpD3TQoz46kcxLdaDLGbhDn+53Fcm0VAe6egvPupZOJNvSHejqQqTTCCYi2ZbuQBdrdDrzKpJtqQ50uVi6bgU6kUyrSqAzs3oze8jMbovT881shZmtN7Pvm1ljOfYzUKNT01Uky6pVo7sCWJuY/gzwBXc/FtgJXFaOndTH0inQiWRbxQOdmc0F3ghcG6cNOBv4UVzkBuCt5dhXvYUqna6lE8m2atTovgh8GMhHn1lAu7v3xulNwNHl2JFqdCICkKvkzszsQmCbu68ys9ZRrH85cDlAS0sLbW1tQy7f09UJGPfdfz9bptePPMM1pKOjY9jyThQqS+1JSzmKqWigA84C3mxmFwCTgOnAl4AZZpaLtbq5wPODrezu1wDXACxevNhbW1uH3NkjP7wb6OLkU0/ntJfNLFshqqGtrY3hyjtRqCy1Jy3lKKaiTVd3v8rd57r7POCdwM/c/d3AcuDiuNglwM3l2F++j04XDItkW61cR/e3wAfMbD2hz+66cmxUt4CJCFS+6XqAu7cBbfH1RuCMcu8jp5MRIkLt1OjGxYELhnV5iUimpTvQxdJp8E2RbEt1oMvlLxhWH51IpqU60OmCYRGBtAc6nXUVEdIe6PI1Ol1HJ5Jp6Q50+QuGVaMTybSUB7rwrLOuItmW6kCXO3B5iWp0IlmW6kCnPjoRgbQHOp11FRFSHujqzKgzXUcnknWpDnQAufo6DaUuknGpD3SN9XX09KpGJ5JlqQ90uXrT6CUiGZf+QFdXp+voRDIu9YGuod50Z4RIxqU+0IWmq2p0IlmW+kDXUFen6+hEMi79ga5egU4k61If6HL1pguGRTIu9YGuMVdHt2p0IpmW/kBXX0dXrwKdSJalPtA1NdQr0IlkXOoDXWN9Hd0KdCKZlvpA15Sro7u3r9rZEJEqykSgU9NVJNtSH+gac2q6imRdNgKdLi8RybTUB7qmXB1dPQp0IlmW+kCnGp2IpD/Q1dfT1+/0aQQTkcxKfaBraghF1AkJkexKfaBrjH/u2qVr6UQyK/2BLqcanUjWVTTQmdkkM7vfzB4xs8fM7BMxfb6ZrTCz9Wb2fTNrLNc+84FOFw2LZFela3RdwNnufgpwKnC+mS0BPgN8wd2PBXYCl5Vrh00KdCKZV9FA50FHnGyIDwfOBn4U028A3lqufTap6SqSeRXvozOzejN7GNgG3AVsANrdvTcusgk4ulz7O9BHp2vpRDIrV+kdunsfcKqZzQBuAo4vdV0zuxy4HKClpYW2trYhl+/o6ODZx9YAsOKBVbRvqB9lrquvo6Nj2PJOFCpL7UlLOYqpeKDLc/d2M1sOnAnMMLNcrNXNBZ4vss41wDUAixcv9tbW1iH30dbWxqtfdTI8cC8nLjyZ1y44oqxlqKS2tjaGK+9EobLUnrSUo5hKn3U9ItbkMLNm4FxgLbAcuDgudglwc7n2mb+OTn10ItlV6RrdHOAGM6snBNkfuPttZvY4sMzMPgU8BFxXrh3m74zQWVeR7KpooHP31cBpg6RvBM4Yj32qRiciujNCRFIv9YGuKRfOtOpeV5HsSn2g0y1gIpL6QNekC4ZFMi/1ge7AME0aTl0ks1If6OrqjIZ6U41OJMNSH+gg1Op01lUkuzIR6Joa6nXWVSTDMhHoVKMTybZsBLqcAp1IlmUi0DXl6nQdnUiGZSLQqUYnkm2ZCHSTGurp1MkIkczKRKBrbqhnf7cCnUhWZSLQTWqoZ7/ujBDJrEwEuubGejp7VKMTyapsBLqGOjVdRTIsI4Gunv2q0Ylk1qgDnZldYWbTLbjOzB40s/PKmblymdSoQCeSZWOp0f2pu+8GzgNmAn8MfLosuSqzyQ05unv76ev3amdFRKpgLIHO4vMFwHfc/bFEWk1pbgzF1AkJkWwaS6BbZWb/TQh0d5rZNKAmr+Fobgj/G6Hmq0g2jeXvDi8DTgU2uvs+MzsceG95slVek/KBTmdeRTJpLDW6M4F17t5uZu8BPgrsKk+2yqu5MQQ6NV1Fsmksge7rwD4zOwX4ILAB+HZZclVmarqKZNtYAl2vuzvwFuAr7v5VYFp5slVezWq6imTaWPro9pjZVYTLSl5rZnVAQ3myVV6TYtN1n2p0Ipk0lhrdO4AuwvV0W4G5wGfLkqsyy9foOlWjE8mkUQe6GNy+CxxmZhcCne6uPjoRqTljuQXs7cD9wB8CbwdWmNnF5cpYOeXPuirQiWTTWPro/h54tbtvAzCzI4C7gR+VI2PlpOvoRLJtLH10dfkgF70wxu2NmwN9dKrRiWTSWGp0d5jZncCNcfodwO1jz1L5NdQb9XWmpqtIRo060Ln7/zWzi4CzYtI17n5TebJVXmYW/zeiJm/FFZFxNpYaHe7+Y+DHZcrLuJqkwTdFMmvEfWpmtsfMdg/y2GNmu4dZ96VmttzMHjezx8zsiph+uJndZWZPxueZoy1QMc2NdeqjE8moEQc6d5/m7tMHeUxz9+nDrN4LfNDdTwSWAH9pZicCHwHucfcFwD1xuqz0l4ci2VXRs6TuvsXdH4yv9wBrgaMJ98veEBe7AXhruffd3FCvW8BEMqpql4OY2TzgNGAF0OLuW+KsrUBLufc3pSnHvq7ecm9WRCYACwOQVHinZlOBnwP/6O4/MbN2d5+RmL/T3Q/ppzOzy4HLAVpaWhYtW7ZsyP10dHQwdepUAL70YCc79jufPKu5jCWpnGRZJjqVpfaMpBxLly5d5e6LxzlL5eXuFX0QRji5E/hAIm0dMCe+nkMY0HPI7SxatMiHs3z58gOvr1z2kJ/16XuGXadWJcsy0akstWck5QBWeoXjxlgfFW26mpkB1wFr3f3ziVm3AJfE15cAN5d731ObcuxV01Ukk8Z0Hd0onEUYv+5RM3s4pv0d4W8Sf2BmlwHPEAYJKKspTTn2dulkhEgWVTTQufuvKP6XiOeM576nTcrR3ddPV28fTbn68dyViNSYmrwJfzxMiUM1qVYnkj3ZCXRNofLa0al+OpGsyUygmzYpBjqdkBDJnMwEunyNbm+3Ap1I1mQu0KnpKpI9mQl005rUdBXJqswEugNNVwU6kczJTKCbqpMRIpmVmUA3pVGBTiSrMhPo6uvC/0ao6SqSPZkJdBCar6rRiWRPtgJdU44O3QImkjmZC3RquopkT+YC3e79PdXOhohUWKYC3YzJDexSoBPJnEwFusOaFehEsihbgW5yA+37e/L/UyEiGZGtQNfcQHdvP509/dXOiohUUKYC3YzmRgA1X0UyJluBbnIDAO37u6ucExGppEwFusOaQ6DbtU81OpEsyWSga1fTVSRTMhnoVKMTyZZMBbp8H51ORohkS6YC3dSmHPV1ppMRIhmTqUBnZro7QiSDMhXoAGY0N9CuPjqRTMlcoJuuGp1I5mQu0B0+pZEX96qPTiRLMhfoZk9tZEdHV7WzISIVlMFA18QLHd3092sEE5GsyFygmzW1id5+Vz+dSIZkLtDNnhpGMFHzVSQ7MhfojpjaBMCODp2QEMmKzAW62dPygU41OpGsqGigM7Nvmdk2M1uTSDvczO4ysyfj88zxzMPsqQp0IllT6Rrd9cD5BWkfAe5x9wXAPXF63MxobqC+zhToRDKkooHO3X8BvFiQ/Bbghvj6BuCt45mHujrj8CmN7NijPjqRrKiFProWd98SX28FWsZ7h7OnNvHCXtXoRLLCKv3Xf2Y2D7jN3V8Vp9vdfUZi/k53H7SfzswuBy4HaGlpWbRs2bIh99XR0cHUqVMPSf/cyk72djsf/53m0Raj4oqVZSJSWWrPSMqxdOnSVe6+eJyzVF7uXtEHMA9Yk5heB8yJr+cA60rZzqJFi3w4y5cvHzT9b3/0iC/65F3Drl9LipVlIlJZas9IygGs9ArHjbE+aqHpegtwSXx9CXDzeO/wqBnN7Ojooqu3b7x3JSI1oNKXl9wI3Au80sw2mdllwKeBc83sSeD1cXpczTlsEgBbd3WO965EpAbkKrkzd39XkVnnVDIfR88IfXPPt+/nmFlTKrlrEamCWmi6VtycGOi2tKtGJ5IF2Qx0sem6uX1/lXMiIpWQyUA3qaGe2VMb2aw+OpFMyGSgA5hzWLNqdCIZkdlAN3dmM8+9uK/a2RCRCshsoJs/ewrPvriP3r7+amdFRMZZpgNdb7+zaaearyJpl9lA9/IjwvVzT+3YW+WciMh4y2ygmz873MC8UYFOJPUyG+hmTm7gsOYGntrRUe2siMg4y2ygMzPmz57Cxu2q0YmkXWYDHcCCl0xl3dY9+eGiRCSlMh3oTjxqOi/s7WbbHo02LJJmmQ50Jx11GACPbd5V5ZyIyHjKdKA7Yc40AB57fneVcyIi4ynTgW7apAbmzZrMY5sV6ETSLNOBDmDh3Bk8sqldJyREUizzge6MeTPZsqtTt4KJpJgC3fxZANy38YUq50RExkvmA92Cl0xl5uQGVjz1YrWzIiLjJPOBrq7OeM38Wfx6/Q7104mkVOYDHcA5J7yELbs6WaPLTERSSYEOOOeEFuoM/vvxrdXOioiMAwU64PApjZwx/3B++ugWNV9FUkiBLvqD0+aycfteVj6zs9pZEZEyU6CLLjxlDtOacnxvxbPVzoqIlJkCXTS5McdFi+Zy6yOb9e9gIimjQJfw562voL7O+MLdv6l2VkSkjBToElqmT+LSs+bxkwef150SIimiQFfgynOO45hZk/nQDx/hxb3d1c6OiJSBAl2B5sZ6vvTO09i+p4v3fXslezp7qp0lERkjBbpBnPrSGXzxHafyyHPtvP2b97F+255qZ0lExkCBrog3LJzDtZcs5re7O3njv/2Kf7njCV7o0H9LiExEuWpnoJa1vvIl3HHla/nkbWv5+s838O+/3MjvLTiC1y6YzenHzOS4lmlMaqivdjZFZBgKdMN4ybRJfPldp3HFOQv4wcrnuP3RLdzzxLYD81umNzF35uT4h9iNzJjcwJTGepoa6mmsr6Opoe7Ac66ujjoz6iyMmjLY63ozLP+6Lrw2gw3tfcx4rh0DzMAI6XnJtPzrgXTisnZgOsyzobdXZH7c1MD+EttK5gXjoPn5dbt6nf3dfQPbO7BdK5geyOfB0weniwzHauXeTjM7H/gSUA9c6+6fHmr5xYsX+8qVK4fcZltbG62trWXLY97m9v089Gw7G7d38MyL+9jcvp/2fT207+tm574e9vf0lX2fMrySAyQHL1hs/nDbw6C3t5eGXO7g+SXmg6L7GT4Ph5R5FHlPTu/ft4+3n3ksV77+OIZjZqvcffGwC9aQmqjRmVk98FXgXGAT8ICZ3eLuj1c3Z4M7akYzR81oLjrf3enpc7p6++jq7ae7t5+u3n76+vvpd+jrd/rd8cTrfic89ydeJ9JXr17NwoULcSc8EvtyQhp4wTy4XvaHAAAGAUlEQVTwRFryR+2geQXz46YK1o3rJfeR2Hdy3UO2V7Duhg0bmP/yVxyYLnzvDl6HQ7Yx2HySeS9h+cL5HDK/tPU2bdrE0UfPPTT/I8wHiflly/shyw8+H4dt2ztpmT6JtKqJQAecAax3940AZrYMeAtQk4FuOGZGY85ozNUxrVzb3JKj9fiWMm2tutr8OVpbX1HtbJRFW9t2WltPqnY2xqytrY3WM15W7WyMm1o563o08FxielNMExEZs1qp0ZXEzC4HLgdoaWmhra1tyOU7OjqGXWaiUFlqU1rKkpZyFFMrge554KWJ6bkx7SDufg1wDYSTEcOdaBivkxHVoLLUprSUJS3lKKZWmq4PAAvMbL6ZNQLvBG6pcp5EJCVqokbn7r1m9lfAnYTLS77l7o9VOVsikhI1EegA3P124PZq50NE0qdWmq4iIuNGgU5EUq9mbgEbKTPbDjwzzGKzgR0VyE4lqCy1KS1lGUk5jnH3I8YzM+U2YQNdKcxs5US7J68YlaU2paUsaSlHMWq6ikjqKdCJSOqlPdBdU+0MlJHKUpvSUpa0lGNQqe6jExGB9NfoRETSG+jM7HwzW2dm683sI9XOz3DM7Ftmts3M1iTSDjezu8zsyfg8M6abmf1bLNtqMzu9ejk/mJm91MyWm9njZvaYmV0R0ydiWSaZ2f1m9kgsyydi+nwzWxHz/P14fzZm1hSn18f586qZ/0JmVm9mD5nZbXF6QpZjNFIZ6BIjFr8BOBF4l5mdWN1cDet64PyCtI8A97j7AuCeOA2hXAvi43Lg6xXKYyl6gQ+6+4nAEuAv43s/EcvSBZzt7qcApwLnm9kS4DPAF9z9WGAncFlc/jJgZ0z/QlyullwBrE1MT9RyjJy7p+4BnAncmZi+Criq2vkqId/zgDWJ6XXAnPh6DrAuvv4m8K7Blqu1B3AzYYj8CV0WYDLwIPAawoW1ucJjjTAoxZnxdS4uZ9XOe8zPXMIPzNnAbYS/i5hw5RjtI5U1OtIzYnGLu2+Jr7cC+bHUJ0T5YpPnNGAFE7Qssbn3MLANuAvYALS7e29cJJnfA2WJ83cBsyqb46K+CHwY6I/Ts5iY5RiVtAa61PHw8zphTpGb2VTgx8CV7r47OW8ilcXd+9z9VEKN6Azg+CpnacTM7EJgm7uvqnZeqiWtga6kEYsngN+a2RyA+Jz/Q9maLp+ZNRCC3Hfd/ScxeUKWJc/d24HlhCbeDDPLD3GWzO+BssT5hwEvVDirgzkLeLOZPQ0sIzRfv8TEK8eopTXQpWXE4luAS+LrSwj9Xfn0P4lnLJcAuxLNwqqy8Aei1wFr3f3ziVkTsSxHmNmM+LqZ0Ne4lhDwLo6LFZYlX8aLgZ/F2mtVuftV7j7X3ecRvgs/c/d3M8HKMSbV7iQcrwdwAfAbQp/K31c7PyXk90ZgC9BD6C+5jNAvcg/wJHA3cHhc1ghnlTcAjwKLq53/RDl+l9AsXQ08HB8XTNCynAw8FMuyBvhYTH85cD+wHvgh0BTTJ8Xp9XH+y6tdhkHK1ArcNtHLMdKH7owQkdRLa9NVROQABToRST0FOhFJPQU6EUk9BToRST0FOhlXZnalmU2udj4k23R5iYyreDX+YndPwz9lyQSlGp2UjZlNMbOfxvHb1pjZx4GjgOVmtjwuc56Z3WtmD5rZD+M9sZjZ02b2L2b2aBwD7thqlkXSRYFOyul8YLO7n+LuryKMmLEZWOruS81sNvBR4PXufjqwEvhAYv1d7r4Q+EpcV6QsFOiknB4FzjWzz5jZa919V8H8JYSBUH8dhz66BDgmMf/GxPOZ455byYzc8IuIlMbdfxOHQr8A+JSZ3VOwiAF3ufu7im2iyGuRMVGNTsrGzI4C9rn7fwCfBU4H9gDT4iL3AWfl+99in95xiU28I/F8b2VyLVmgGp2U00Lgs2bWTxiF5c8JTdA7zGxz7Ke7FLjRzJriOh8ljDIDMNPMVhP+q6FYrU9kxHR5idQEXYYi40lNVxFJPdXoRCT1VKMTkdRToBOR1FOgE5HUU6ATkdRToBOR1FOgE5HU+1+rRyHTq1rxGgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learning_rate = 0.03\n",
    "iterations = 400\n",
    "initial_w = np.random.randn(train_X1.shape[1])\n",
    "w, weights_history, cost_history = rl_functions.batch_gradient_descent(train_X1,\n",
    "                                                                       train_y,\n",
    "                                                                       initial_w,\n",
    "                                                                       learning_rate,\n",
    "                                                                       iterations)\n",
    "plots.simple_step_plot([cost_history],\n",
    "             \"loss\",\n",
    "             'Training loss\\nlearning rate = {} | iterations = {}'.format(learning_rate,\n",
    "                                                                          iterations))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R² score = 0.6770\n"
     ]
    }
   ],
   "source": [
    "prediction = rl_functions.linear_regression_prediction(test_X1, w)\n",
    "r_2 = util.r_squared(test_y, prediction)\n",
    "print(\"R² score = {:.4f}\".format(r_2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing stochastic_gradient_descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAToAAAEoCAYAAADIXD3MAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnXecHVX5/9/PtmySTSVkE0IJvUgksAGCoGYRIk0Rwa+CBQQJFkR/FgQRBUFEUUClKF9Qyhey1CiEUJKwIdIC2fSekF43fXezfff5/XHO3Z292XI3e8vmzvN+veZ1Z86cM+eZO2c+c/oRVcUwDCOdyUi1AYZhGInGhM4wjLTHhM4wjLTHhM4wjLTHhM4wjLTHhM4wjLTHhM5oFRFZLyIqIo92MtxRPpyKyDcSZV8r8d7p46xPVpzG/oMJ3X6EiKwOiEhb221xim4WMAP4uJPhqn24GcDWONliGF0iK9UGGJ1iNrDZ7x8MDPP7c4Aav7++rcAikqOqtbFEpKpf3BcDVXU9MHpfwhpGorAc3X6Eql6iqqNVdTQQLFI2uavqoyJyTiCHd42IvC0i1cA4ETlcRF73RdMqv80XkRuCcUUXXaOKpD8WkWdEpNz7uzkQbq+iq4h8J+D2BRH5r493sYicHxXvpSKyXESqvd0XdaUo7O9/lo9vj4i8LyKXRvn5qYgs8ufLRGRBsMguIqNFZIqIbPN2rRWRV0Xk5M7aY6QGE7r05yHgBGAVoEA+MNbvLwbKgBOBv4jIdTFe84/AZ3G5yGHAXSJSGGPYF70NAMcBRSLSH0BERgLPAUcBtcAQoCjG6+6FL8Y/CpwMlOLudTTwgohc4/1cAvwJOB5YjcsRHw5ERDoTmAR8DqgHFgE9gAuAY/fVNiO5mNClP/8FDlbV44G/A8uAw1X1EFU9BTgIeM/7/VqM15wBDMcJaKTy/3Mxhr1PVY8Bvu6P+wKj/P6NuDRZARyvqscCD8d43RaISB/gJn84ASdeh9F8r3eKiADH+OM3VPUTqnoC0B/3MQAYBAzw+yer6imqmo8Tuci1jG6OCV3683dVrQFQ1QZcTulmEVkjInU4ofqU93tQjNd8VlXrVLUU2O7d8tsLEOAp/7so4BYJe6L//a+qbvD742O8bjQjcDkvgCJVbfT1ky96tyG4es7XgTrg8yKyVUTeBe7H/U+o6hbgQx9mpS/WPgd8Gti0j7YZScaELv3ZEnX8N+A64FBgJS53ts2fy4zxmrsC+5EcnXQybLAbSHTYpE2po6pzgU8AtwHv4MT++8B/A3VwY4BrccXoCuDLuCLx3cmy0+gaJnTpT7RoRFpEJ/miYSHNLbmpZr7/PUtEhvj9y7twrUhL9FdFJENEcoBIQ8RmYL2IHAM0qOrtqnoJrt5wD65Hwmd8Hd0ZwL9U9du+Iehxf42z99E2I8lY95LwMQ/3Mp8vIkuBgXSfD949uHrCvsBSEdmMK152GlUtF5G7gd/gcmCrgGxgqPfyK1VVETkbeFhENuJyv0OA3t7PPB9mKlAuIuuARlwOMHLe2A/oLgncSB4/Al7B5VrycMWv11JqkUdV5wD/A6zA1a+V4oqMEao6eb3bfPjZwGBcI8MHwGWq+pj3VoJrrKjFtbz2xnWW/raqFuPq7/6BK+YfBBwNrMW1Zv+ws/dopAaxGYaN7oSIHKOqywLHtwO/9odHq+qK1Fhm7M9Y0dXobpSIyApgDa7BJNIg8JiJnLGvWI7O6FaIyJO4Vs58XLFxCa7y/2HfPcYwOo0JnWEYaY81RhiGkfZ0KHSBqYEeT4I9nUZExgQGfY9JtT1GYvHp8fEO/Dzu08Pq5FjVNURkmrd3WqptSVfSIUdXRvP8Z2UptqVTJGAOubggIgV+hpMyEakUkXdF5NwYw54jIu/4cGUi8oaIFET5+auILPGzn1R58XpMRA6L0y18jEsPswNxplT8RGR44HlfFXV6Ec7eRXuH3P8RkStFZIaIVPgZYpaIyE+i/MSSbrJF5DcislJEasXNnHO/H9fcPqra7oab0UGBxzvyG68NyElWXHG2Owtf7xmjf/Xbbam2PWDTJ3F97BQ3ceZ6v18PjO0g7Oe9P/Xhtvr9PcCIgL+VuFbVEmB54H9YEoN9q/clLeIaNBRYHcf/KuZ0ipsEIXKfV6X6OScxPd0XuO+NuD6KG4BX9iHdPOXdG3CNVLX+eBqQ0a4dMSasFkIH9AHuxfU2r8UNbv470D/g5zzczBml3k+ZPz6/jYd/I/BvoBI3qHpMMGEAE/25VcA1gWsE/Y3xbrcF3Ar9n1vlf0dH3d91uA6glT6Ob0Rfr6MXx9u3Ctdrvj/wTdxA8G24lsOdwBvAaa3YHNxWB64/FnjL/2+RWXu/kISE+bK3ZZV/zlm4TrYKzOsg7Dzv730frg/N00O9HPCXGxXuqcB/cEAM6fHxDvw0PZuoNBy9RdLLENzY1Q24tLoG15G6R+Ca02h+qX6Be2l3+nM/w01+usM/763AS8Ax/vxVbcQ/Lfragfh6Ar/DdZ6u9dd+BTgl4Cd43YuB6bh0vgS4KOCvN/AgLp1X4yZimAH8JMFp6fSAfd8jkAkA+nQm3QCnBK51vXf7QsDty3EVOiAH9yVW3FjCubiBzurdswMPvxZXjJgFlHs/dcBJrQhdDbAbN0bxT7QUg1p/47tpVvTjYhS6av/g62gWpizv74KAv+24nEZF9PU6eJlqvT1LceMn+wMP+AS3FPcCVHu/ZbiX6hSaxSPyFfsAmOCvfRlONBVYR3OupxHXqz+W59XWtrqdsFk4wVfgHwH3XwbCH9RG2GEBPzcH3B/xbpVAZsD9x7iXLZijW0gHOWL2Tegm0JxLqPH/9Qf+ORwQ+M8qcOm5xh8Hcx3TAuHrva0r/bmJPuwiXPqN5E7WAbnAhbhidOQ+P/bxP9SO0E0O+I/MGxj5H0e2InS1uCm4Is+vDBjo/f05YPssH38dMKWD/3Ea7acl7SD8fQFbxuPesY3++RzYmXQD3BLwN9T7ycC9Zwo8Em+h+xbNgvUJ73ZY4OF+PSBiwRzegMDDuqMVoVsc8e9vbEzg3PO4GS4+GXD7boxC90PvdkPALSKS0/3x2kDcz0Rfr4OXSYHrvJv47RigV8DvUQG/wdxoxO22qGuv9O5P09wF6H+927IOntcEml/k1rYJ7YQdErDpjoD7NQH30W2EHd3GPd4RcB8ScL8/4K7ATNoQ0a4KXVtu3v3XNH/oIi/QmQG7zmzlpT8vkk797yfwH3h/fE7A7+daSetXtSEo0/xxYcDvzwLPZqd3e7EVofuzd/tiK3a+4o9vDcTZFzi1g//xoQ7S0gcdhJ8UsKUWWECzTnyAe89jSje4EmPkOPjBXOfdXm/Pln0ZGXG6/80CFri5C1swGveC5gCPi8incF/NYMNHa/OePaGquwBUtSHquk+rqopIa3OYdURb858toXn+s9cjceOm4unMjBlVOBFynzfAz5j7oK9M7U/LaYjanfNNRA7ETRIJcAVwRdR/cbSIHKCq2/cK7Gy4pBO2x0qsUzDFHFZVfywiP8WNHf07bsbi/xORczW5HYMj6XkgsLGN9Pxu4Hipqr4OLp16t0OBf4jIJ3Hjh2N+3m1wamD/GR/XZhEpBi6heaLSIO3N8/cKcBHwWxG5Fpfz+4CW0/Hvhap+v/OmtyCoL1ep6jMi8l3cZKqn0/I+o4k1zcXkrytDwOpw2eBoIvOfvYrLydTjsvPVuOE8ObQ+71n0vGlBIgJYH0iIMd1gQMASNf/ZVlVtbLqwSB6uPq4/7p5n4/6ryAsV65xv4Irrpa24Z7cVQEQm0DxDR2tsakcMt+GEuyduEHyE4P7aNsKua8N/ZL+KqFXBvFAsEZF7cUJXiJup+M127E8UFbjiaDS7oo5bpFMROQJXt5yDq54pwb1XI72XzjzvrtBmOlfVR0RkCS63NwIowP3P3/Zji/e0dkEReQhXvG8TddNWtcWGwP5M//tRwG04rt4+QnvpJjp9bRKRDFwmCtpOl8C+CV3E0Czgx6r6AYCIZAHnAotF5ACcyAH8WlV/LyLDcbmotuiK2Owr84HPAGNFpI+qlhP7dOIRou0+FidyAFer6ngRGY2raI0mIiqRaYFQ1a2+C8RwXFb/UlWtAxCRQ3HTebc3f9zJuKqEtljT5o24D8lU3Nd/rG+2r8K9IADzVXWjt+VJ4DTgQ1X9lqpuEJEFuFzyF0XkHn9vkW4pU3xOfQR+Zl+fS8/A1ZVG6E1iqPS/vUREIrlvXHqO1NV+Q/14WhHpiatbmxJ1nejnHfl4A3xeVd8Xka+x98zIlYH9ju4xKAZXAH/y8/NF1uWYuXeQthGR04CFqjrdH0fS40G4KbtK2gh6As0f6H3hTVzxGpxgLqNlbnRZJ9LN68Cd3u1SXD34hbg6UHAzRbdNjHUiwTq6HjRXrDbivoKLae6SMAb3JYmUnWtxgrKD5or+yLWG03a9xZjAuTEB9xb1Wq35I1BH1971aNkYsQ1XN7antXhjqQfy7gMC91mJa1HaHG239zvLu9XgEvdd3v2rUXbNxlXiNhKosE7EBpxEc4V2sHtJA77OR1upV/Ju53t/SstuApU0N0B9ybuV4Sr+g//NWgKtce2kx32powvW0S7FFd164taEWEtzWp2HayCJNCANb+t+vftxNNc7RRrTtgbiuipSo+GfpeJyfjNorj9u7b+MboyINMRV0XpjRMTO4a3E/X+4UsUqnKhFrlVBoB49AWkp2GJf4/+bSKNgsBW+w3Tj/UXqzxv8fxLpXjKdDrqXdLrDsLr1B8bgupesxtWxHIgTvDuBBeqsuhT38jbgsu9fp3nK7m6Bqk4CvosT5d64F+BnAS+dmv/MX3Mn8BVcXUkG7mF8oQ3vN9A8q+4o/EItqvos7uG/hcstHI978Z7HtUgnDHVTi38W96Ll4ooG7wMXqK+baifsa7iPx3s+XC4uR/RZf11wIvIKTuiOw30YVuDqbc5Ql6tOBP/ErRexG/c/n46r1N6Gq4d7FFdNcDyuov4jXGtze1UqqOoS4GqciOTg0vhedbz+nbgWd689cbnh9nLeXwTuwn18j8R95CbiGkfmxHLDAV4F3sZlUkbgW1xxXb2ii+ZxQ1XrcX3kHsQ19hyFe/634HoWRPzFkm4ArgR+i/swHYn7r/+G60rTSDuEelC/iGTjVshaFXB7DJdwa3FN4PvVaIt0xxfrp6nqVSk2xdiPCPt8dL2BFSJSgisaHoP7ogP83kTOMNKDsAtdNa44cCqulawa15XgH6r6VHsBDcPYfwh10dUwjHCQDrOXGIZhtIsJnWEYac9+W0c3aNAgHT58eIf+9uzZQ+/eieqDGhvdwQazw+yIlx0lJSXbVPXAJJgUPxLZ+TSRW0FBgcZCcXFxTP4SSXewQdXsiMbsaEmsdgAztRtoQGc2K7oahpH2mNAZhpH2mNAZhpH2mNAZhpH2mNAZhpH2mNAZhpH2JFXoRCRXRD4UkbkislBEbvfuj4vIKhGZ47eRHV3LMAwjVpLdYbgGOFtVK/wUSe+IyGv+3M9V9YV4RvbU+6vZsLGeMfG8qGEY+x1JFTrf2bDCH2b7LWGzCjw9Yy09G+s79mgYRlqT9Do6EckUkTm42Vwnq+oMf+p3IjJPRO4TkR7xiCszQ2i0yVkMI/SkbJomvyTgBOCHuGmWN+Omon4E+FhVf9tKmHHAOID8/PyCoqKiduO4/b0qemY2cOPpeXG2vnNUVFSQl5daG8wOsyNedhQWFpaoamtLLnZfUjn+DLd48M+i3MYAEzsKG8tY14sfeEcv/ONrHfpLNPvbWMZEY3a0ZH+zAxvr2j4icqDPyUWWkzsXt67nUO8muFWiFsQjvswMoTElqygahtGdSHar61DgCRHJxNUPPqeqE0XkLb9CvQBzcCtzdZlMsTo6wzCS3+o6D7fgb7T72YmILyMDEzrDMNJ7ZIS1uhqGAWkudBlWdDUMgzQXuswMIUW9ZwzD6Eakt9CJ0JhqIwzDSDlpLXQZVkdnGAZpLnSue4kpnWGEnfQWOsvRGYZBmgudFV0Nw4A0F7pMsQ7DhmGkudBZjs4wDEhzocsUsSH9hmGkudBZjs4wDNJc6FzR1ZTOMMJOWgudTdNkGAaku9BZ0dUwDNJc6Gz2EsMwIM2FLjMDG9RvGEZ6C531ozMMA9Jc6DLF5qMzDCPJQiciuSLyoYjMFZGFInK7dz9cRGaIyAoReVZEcuIRnzVGGIYByc/R1QBnq+pJwEjgPBEZDfwBuE9VjwJ2AtfEI7IMPzJCLVtnGKEmqULn17+t8IfZflPgbOAF7/4Ebm3XLpOZIQA0WLbOMEJN0uvoRCRTROYApcBk4GNgl6rWey/rgWHxiCsidKZzhhFuJFXFOhHpD0wAbgUe98VWROQQ4DVVPbGVMOOAcQD5+fkFRUVF7cYxcWUtLyyr45Fze5GTKfG+hZipqKggLy8vZfGbHWZHPO0oLCwsUdVRSTApfqhqyjbg18DPgW1Alnc7A3ijo7AFBQXaEQ9PW6GH/WKiVtbUd+g3kRQXF6c0/ghmR0vMjpbEagcwU1OoG/uyJbvV9UCfk0NEegLnAouBYuAy7+1K4D/xiM+XXG1gv2GEnKwkxzcUeEJEMnH1g8+p6kQRWQQUicidwGzgsXhEliGROjoTOsMIM0kVOlWdB5zcivtK4LR4xydijRGGYaT5yIimoqspnWGEmjQXOiu6GoaR7kJn/egMwyDdhc4XXdVydIYRatJc6CxHZxhG2gud+7U6OsMIN2ktdGKNEYZhkOZCFym6ms4ZRrhJc6Fzv5ajM4xwk+ZCZ40RhmGkudB5nbOJNw0j5KS10EUm3rR+dIYRbtJa6KzoahgGpL3QuV9rjDCMcJPWQmf96AzDgDQXOutHZxgGpL3QuV/L0RlGuElzobPGCMMw0lzoxHJ0hmGQZKETkUNEpFhEFonIQhH5kXe/TUQ2iMgcv10Qj/iacnSWpTOMUJPsVcDqgZ+q6iwR6QOUiMhkf+4+Vf1TPCOzoqthGJD8VcA2AZv8frmILAaGJSq+DJ9ftaKrYYSblNXRichw3NKHM7zT9SIyT0T+KSID4hGHLY5jGAaApGIcqIjkAW8Dv1PVl0QkH9gGKHAHMFRVr24l3DhgHEB+fn5BUVFRu/Es3dHA7z+s5sZTcznhgMx430bMVFRUkJeXl7L4zQ6zI552FBYWlqjqqCSYFD9UNakbkA28AfykjfPDgQUdXaegoEA74qNV2/WwX0zU6ctKO/SbSIqLi1MafwSzoyVmR0titQOYqUnWja5uyW51FeAxYLGq3htwHxrwdgmwIE7xAdYYYRhhJ9mtrmcC3wTmi8gc7/ZL4HIRGYkruq4GrotHZDYywjAMSH6r6zuAtHJqUiLiax7rakJnGGEmrUdGNHcYTrEhhmGklLQWuqap1C1HZxihJq2FzoquhmFAmgtdZM0Ia3U1jHCT1kJnra6GYUCaC531ozMMA9Jc6CI5OqujM4xwk9ZCl53pbq+m3vqXGEaYSWuh65Pr+kOXV9en2BLDMFJJWgtdXo+I0NWl2BLDMFJJWgtdVmYGPTItR2cYYSethQ6gZ5ZYjs4wQk7aC12vbMvRGUbYSX+hyxITOsMIOWkvdD2zhDIruhpGqAmB0FnR1TDCTtoLXS9rjDCM0JP2QtczWyizHJ1hhJr0F7osqK1vpKa+IdWmGIaRItJe6HL8yP5aG+9qGKEl2csdHiIixSKySEQWisiPvPtAEZksIsv974B4xZnl79CEzjDCS7JzdPXAT1X1BGA08AMROQG4CZiqqkcDU/1xXIgIXV2DTdVkGGElqUKnqptUdZbfLwcWA8OAi4EnvLcngC/FK85mobMcnWGEFUnVpJQiMhyYDpwIrFXV/t5dgJ2R46gw44BxAPn5+QVFRUUdxjNtVQWPLxXuOqsnB+WlpkqyoqKCvLy8lMRtdpgd8bajsLCwRFVHJcGk+KGqSd+APKAE+LI/3hV1fmdH1ygoKNBY+OP4yXrYLybqoo27Y/KfCIqLi1MWdxCzoyVmR0titQOYqSnQja5sSc/iiEg28CLwtKq+5J23iMhQf34oUBqv+LKt6GoYoSfZra4CPAYsVtV7A6deBq70+1cC/4lXnNbqahhGVpLjOxP4JjBfROZ4t18CdwPPicg1wBrgf+IVYaZfCazWcnSGEVqSKnSq+g4gbZz+XCLizLbuJYYReva56CoiPxKRvuJ4TERmicjYeBoXD6zoahhGV+rorlbVMmAsMABXJL07LlbFkSw/BMwaIwwjvHRF6CJF0AuAp1R1IW0XS1OG5egMw+iK0JWIyJs4oXtDRPoA3U5N8rKFzAxhxqrtqTbFMIwU0ZXGiGuAkcBKVa0UkYHAt+NjVvzIyxFOHNaPdTuqUm2KYRgpois5ujOApaq6S0S+AfwK2B0fs+JLz+wMK7oaRojpitA9DFSKyEnAT4GPgSfjYlWcycnKpMYaIwwjtHRF6Or9uLeLgQdU9UGgT3zMii85mZajM4ww05U6unIRuRnXreTTIpIBZMfHrPjSIyuDWptK3TBCS1dydF8FanD96TYDBwP3xMWqOJOTlWFDwAwjxOyz0HlxexroJyIXAdWq2j3r6KzoahihpitDwP4H+BD4Cm4Q/gwRuSxehsWTnCwTOsMIM12po7sFOFVVSwFE5EBgCvBCPAyLJyZ0hhFuulJHlxEROc/2Ll4vYWRnWh2dYYSZruToXheRN4Dx/virwKSumxR/crIyqGtQGhuVjIxuNxzXMIwEs89Cp6o/F5FLcZNpAjyiqhPiY1Z86eFH9tc2NJKbkZliawzDSDZdmnhTVV/Erf/QrcnJdEJX19BIbrYJnWGEjU4LnYiUA61N1yuAqmrfLlsVZyLF1UarpjOMUNJpoVPVbjnMqz0yfbVcQ4rWsDUMI7UkexWwf4pIqYgsCLjdJiIbRGSO3y6Id7yZPkfX0GhCZxhhJNndQR4HzmvF/T5VHem3uLfcNhVdLUdnGKEkqUKnqtOBHcmME5rXjai3HJ1hhJLu0sH3ehGZ54u2A+J98QyJNEaY0BlGGBFNcnFORIYDE1X1RH+cD2zDteTeAQxV1avbCDsOGAeQn59fUFRU1GF8FRUVzN3dg/+dX8sfPt2T/N7J1/aKigry8vKSHq/ZYXYkwo7CwsISVR2VBJPih6omdQOGAws6ey56Kygo0FgoLi7Wf89er4f9YqKuKC2PKUy8KS4uTkm80ZgdLTE7WhKrHcBMTbJudHVLedFVRIYGDi8BFrTld1+xoqthhJsujYzoLCIyHhgDDBKR9cBvgDEiMhJXdF0NXBfveCONEdaPzjDCSVKFTlUvb8X5sUTHG+leUt9gQmcYYSTlRddkkCnWj84wwkw4hM5GRhhGqAmF0NnICMMIN6EQukjR9d0V26mus2UPDSNshEPofI7u3snL+OWE+Sm2xjCMZBMqoQNYurk8hZYYhpEKQiJ0zfvWHmEY4SMUQhcZGQFEhpoZhhEiQiF0mbbyl2GEmtAJnWXoDCN8hE/oWl3XxzCMdCYcQieWozOMMBMKoQvonI2OMIwQEgqhq603cTOMMBMKoatraF652iTPMMJHKITukIG9mvat5GoY4SMUQjewdw5D+uYC1mHYMMJIKIQuiMmcYYSP0AhdpOXVMnSGET6SKnR+gepSEVkQcBsoIpNFZLn/jfsC1gCRHibWYdgwwkeyc3SPA+dFud0ETFXVo4Gp/jjuiNgCOYYRVpIqdKo6HdgR5Xwx8ITffwL4UiJtqK1v7NiTYRhpRXeoo8tX1U1+fzOQn4hIjh3SB2jO2RmGER4k2d0tRGQ4MFFVT/THu1S1f+D8TlVttZ5ORMYB4wDy8/MLioqKOoyvoqKCvLw8quqVn0yrZECucNdZvToMF08iNqQas8PsiIcdhYWFJao6KgkmxQ9VTeoGDAcWBI6XAkP9/lBgaSzXKSgo0FgoLi5u2r9h/Cz97B/fiilcPAnakErMjpaYHS2J1Q5gpiZZN7q6dYei68vAlX7/SuA/iYooKyODOmuMMIzQkezuJeOB94FjRWS9iFwD3A2cKyLLgXP8cULIyRJqG6wxwjDCRlYyI1PVy9s49blkxJ+dmdFigL9hGOGgOxRdk0Z2Zgb1DcqK0nJq6m0ha8MIC6ETuoqaes65dzq3TFjQcQDDMNKCkAldcx+6Gau2p9ASwzCSSciErvl2bXC/YYQHEzrDMNKekAmdDf8yjDASKqHrkRXM0VmWzjDCQqiELi+3udugyZxhhIdwCV2P7KZ9y9AZRngImdAldSCIYRjdhFAJXZ8WRVfL0hlGWAiv0JnOGUZoCJXQHZDXo2m/tLyG3ZV1KbTGMIxkESqhi66j276nJkWWGIaRTEIldAB9A8XXDFs/wjBCQeiELjc7s2m/vtHmpjOMMBA6oeuR3XzL9Y3WImEYYSB0Qje4T27Tvi1mbRjhIHRC99DXT+H0wwcC2LTqhhESuo3QichqEZkvInNEZGai4snvm8v1Zx8FQIMVXQ0jFHS3MVGFqrot0ZFkZrjWVlv60DDCQbfJ0SWTyASc1upqGOGgOwmdAm+KSImIjEtkRJEc3U4bGWEYoUC6ywSUIjJMVTeIyGBgMvBDVZ0e5WccMA4gPz+/oKioqMPrVlRUkJeX18Jt9e4Gbnu/GoD7xvRkQG5i9b41G1KB2WF2xMOOwsLCElUdlQST4oeqdrsNuA34WXt+CgoKNBaKi4v3clu0cbce9ouJTVuiac2GVGB2tMTsaEmsdgAztRvoRGe2blF0FZHeItInsg+MBRK28GpWhg39Moww0V1aXfOBCeLGnmYBz6jq64mKLCuzW+i7YRhJolsInaquBE5KVnyWozOMcBHKrE2WLXtoGKEinEKXEcrbNozQEso33taLMIxwEUqh65ub3eL4kekfp8gSwzCSQSiFLjc7k9FHDGw6vmvSEu6fsswG+RtGmhJKoYPmYWAR7p+ynHdWJHw+AcMwUkBohe4Pl35yL7dsa401jLQktEJ38IBe/OHSES3cusmwX8Mw4kxohQ4gr0fLRomq2oYUWWIYRiIJtdAdOrBXi+OtFbbOq2GkIyZ0AW5+aT479tSmyBrDMBJFqIWuX69sLjnHpvs7AAAQVElEQVR5WAu31xZsatpfvKmMl+duTLZZhmHEmVALHcDFIw9qcbxy6x4aGpX6hkbO/8t/uWH87Bb962rqG2i0/naGsV/RLWYvSSUHD+jZ4vj5met47J1VLdzW7ahk+KDeVNU2cPyvX+dbZxzGby8+MZlmGobRBUKfoxvW39XTnTC0L0ce2Juy6vq9/KzavgeAhRt3A/Dk+2uYsmhL8ow0Ymbyoi1s841KZdV1lJZXp9giozsQeqHrmZPJC989g/HXjm5aHSyazburqa5r4KaX5je5fefJ1pee3VJWzV+nLo9MCb8XW8tja9mdu25XSoekqWqTYHRXqusamLd+V9NxWXUd1z45k+ueKgHg8/dN57TfTY1bfDe9OI+rH/8obtdLNiVrdvDGws2pNiMlhF7oAEYNH0i/XtnU1Le+/OHiTWV8+aH3WFFa0cL9rSVbWLalnNXb9lBd18C1T87k9Lumcu/kZSzcWEZDo9LYqJRWuuu+Mncjp/5uCrPX7mzXnsWbyrj4wXf5/aTFe51TVXbuqWVreQ0bdlXFdH8bd1Vx/K2vs6Ys9n6CRR+tY9SdU1i+pbxNPyu3VrDTt1L/Z84Gnpu5jqIP18YcR1e5ZcICvvjAu5SWuVzb9gpnyxqfA9+027mv2raHMfcUs6Vs79zdtKWlTTn1jij6aB1vLSkF4OW5G1m3o3IvP9V1bf/Hc9btYunmtv/P9mhsVF6bv6lL9cOXPvw+1z1VEso6ZhO6AH1yXZXl+ScOaXLrlZPJk++vYdGmsr38X/34TMbeN50xf5rGlMVbmBwozt4/ZTlH/nISR/xyEjdOr2LdjkqK/UsSndjX7ahkd1Xz0os/HD8bgEffWcWLJetb+H3qgzWcfMdkTv3dFM68+y2q6xqoqKnfK/e3bEt5kwhNXbyFqroGJq2si/lFi7zQy724766qo66h5Yfg7D+/zXl/cQu1/ahoDje+MI+bXppPebW7l/dWbOOj1TvYXVnH28u2NomAqtLQqHywcju7KmPvzrO1vIbJi7awvaKGSfM3MXPNDqB52cpIDjQnKmf+wFsrWL29kknzN6GqvLVkCw2Nyp465ap/fcSFf31nr7hWbq1o+k9nrd3JFx9o9lNd18AN42fztUc+aBGmZM1Ojrv1dd4LjJn+17ureNPnor704Lt8/v4WC9vtxY49ta1+wF4oWc/3np5F0Ufr2g3f2Kgt0lJrrNxW0e75dCT0jRFB/nb5ydwxcTFHDXZLvl15xmGcdEh/fvLcXADev/lsHvvvKh6NaqwAuP6Z2S2OpyxuWYf3uT+/Ta0XiiffX8OA3jmMPSGf9Tur+PQfiwF48/99hmdmrG2Rc3x1/iYuOmko1z8zm+IlpdRHCdqI296grkH5SsHB3HLh8VTVNfDByu38v2fncsLQvvz7B2c25VRnbG5oetGeuPo06hsaufXfC8jKzOCTB/fjq6cewtLN5ZRX1zdNN1+yZid3TVrM+p3NL98j3yzgqQ/WALClrIZ/vN1ymqu1Oyrp3yuHKx6d0cL97OMGo6oUL62ENyY1uX90yzmUVdfx3Mx1XHzSMI4anMcbCzfzw/GzKRo3mjnrdjGwdw43vjAPgCMP7M3HW/c0hf9w9Q6Wl5ZT9KETgY27q3kh8IF4cZbbv2vSYm5/ZREAQ/rmsjmQw3uxZD1fOnkYH6zczgsl65kwewMnHdyPJ68+nXeWb2Pe+uZc331TlgGwYVcVw296lZ9//li2ltfw/EwX/5uLtnDrfxYgIk3PcvXdFzaHn7yMh6atYHCfXIb0y6W+spo+h+/k5EP684W/vcOGXVXM/fVYauobGNw3l9r6Ru55cykAz85cxxWnH9p0reIlpby5aAuT5m8iK0M4eEBP5q7fzexbz+XluRs5Jr8Phwzsyd2vLWkKM23pVnr3yGLVtj0cPbgPv3l5AbdedALpTHda1/U84C9AJvCoqt7dnv9Ro0bpzJmt15MFmTZtGmPGjOmULbPW7uTLD73Hf35wJicd0p/nZ67j6Pw+jDykPwDvfbyNK/53RgdX6ZgRw/oxf0P7xaae2ZlUtVMcShRnHHEA76/cnvR405WsDNnrIxXNEYN6s3LbnhZug/v0oDSqXve7nz2Sb585nI27qrjkoff2yZ7jhvRhSSB3n5udwfc/mc0NXzmnw7Aist+t69otiq4ikgk8CJwPnABcLiIp+8SccugAVv3+Ak7ywvaVUYc0iRzAp44cxKs3nMXZxw1ucuvXM7vV2U+i1+E58sDeTfsdidxB/XJTInJAqyIX3RUnHvz24k+0OP700YPiev3eOZkd+hlz7IGtuvfIit/r0ZbIPf7tU7noCDfmeuW2PQw/oBdnHdX8H0SLHMDf3/6Y0++aus8iB7QQOYDqukZml6bvWO/uUnQ9DVjhVwNDRIqAi4FFqTLIL73YJp84qB//vOpUtlfU0LenS6jz1u/m0offo1dOJuOvHc2kBZv4+dhjOf+eN7j2nBM5/8QhZGdm8OxH67i04GCmLt7C2u2VbK2o4aD+PZk0fxO5WZl8uNrVPb35k89S9OFa7nx1MbdccDznnTgEVRjQO5sRt73JqcMHcPlph3L80L70ysl09Vd7alm/s4pX5m7k1otO4MIRQ1GUQXk9mPb227y4oS9z1+9qqqj/1hmHMWJYPz511CCe/mAN76zYxtbyGjbtruan5x7D8yXrueL0Qxlz7IEcN6QvjY3Ky3M38nzJOi495WBGDOvHL16cx9GD+/CsL7pdMGIIpWU13HzBcTz30Xry+/bgr2+taPH/fWP0ofxs7LH075XD8UP7smZ7JZcVHOz/x10s2FDGLyfM55zj8+mbm8VLszfw/TFH8taSUpZsLucvXxvJE++tZnCfXI4c3JtLTh7GhX99hwG9cthVVctZRx3INWcdzieG9WVPTT1n/P4tLh55EEcMyqNBlb9OXQ7AjecdyzdGH8bSzeVMmL2BGz9/LJt2V5OdmUFNfQPrdlSybEsFOVkZXDhiKPdPWc6Ls9Zz1lGDePLq06htaGRnZS27KuvYXVXHh6t2MGn+JnrmZHLLBcdzxIF5nHvv24w5djAnDuvLJScPY9W2Pfz+tSVcPPIgxhw7GDbl8J3zT3ONUCMPoldOFre/spBtFbUcMziPzWXVVNU1MPaEIby9rJQFG8o4cVhfZq7eyUmH9G8qpj9z7en0yMrkL1OXM33ZVr5++qE8PcM1Dl1z1uGUltdwxWmH8uKs9by1pJQHrjiZIX1zARjarycz3vtvl9+bbkuqV9D2RefLcMXVyPE3gQfaC1NQUNDRguKq2j1WQe+sDY2NjS2Od+6p2cutPRoaGrW2vqFdOxZu2K31Da1fs6q2Xqcs2typOFVVZ6zcrks3l7V5/o0Fm/Tfs9frb558U9fvrOzUtSNs2V2lL8/Z0OlwtfUN2hC435mrd+hzr07dJxuWbi7Typr6fQrbGl1No+t27NHpy0pbPbdya0W7z2Rf7ABmajfQjc5s3aKOTkQuA85T1e/4428Cp6vq9VH+xgHjAPLz8wuKioo6vHZFRQV5eXnxN7oTdAcbzA6zI152FBYW7nd1dClXWi+0ZwBvBI5vBm5uL0w65+gShdnRErOjJemco+sWjRHAR8DRInK4iOQAXwNeTrFNhmGkCd2iMUJV60XkeuANXPeSf6rqwhSbZRhGmtAthA5AVScBkzr0aBiG0Um6S9HVMAwjYZjQGYaR9pjQGYaR9pjQGYaR9nSLDsP7gohsBdbE4HUQsK1DX4mlO9gAZkc0ZkdLYrXjMFVtfYBwN2W/FbpYEZGZmuJe3N3BBrPD7Nhf7EgEVnQ1DCPtMaEzDCPtCYPQPZJqA+geNoDZEY3Z0ZLuYkfcSfs6OsMwjDDk6AzDCDlpK3Qicp6ILBWRFSJyU4Lj+qeIlIrIgoDbQBGZLCLL/e8A7y4i8ldv1zwROSWOdhwiIsUiskhEForIj5Jti4jkisiHIjLX23C7dz9cRGb4uJ71s9QgIj388Qp/fnhXbYiyJ1NEZovIxFTZISKrRWS+iMwRkZneLRXpo7+IvCAiS0RksYickQo7UkKq54lKxIabAeVj4AggB5gLnJDA+D4DnAIsCLj9EbjJ798E/MHvXwC8BggwGpgRRzuGAqf4/T7AMtwaHEmzxV8rz+9nAzP8tZ8Dvubd/w58z+9/H/i73/8a8Gycn81PgGeAif446XYAq4FBUW6pSB9PAN/x+zlA/1TYkYot5QYk5Kb2YSLPOMQ5PErolgJD/f5QYKnf/wdweWv+EmDTf4BzU2UL0AuYBZyO64iaFf18cFNzneH3s7w/iVP8BwNTgbOBif6lTYUdrQldUp8J0A9YFX1P3SGdJmNL16LrMCC40u9675ZM8lV1k9/fDOT7/aTY5oteJ+NyVEm1xRcX5wClwGRc7nqXqta3Ek+TDf78buCArtrguR+4EYisvH1AiuxQ4E0RKfHLAUDy08fhwFbgX74o/6iI9E6BHSkhXYWuW6Huk5i05m0RyQNeBH6sqmXJtkVVG1R1JC5HdRpwXCLjaw0RuQgoVdWSZMfdCmep6im45Tx/ICKfCZ5MUvrIwlWvPKyqJwN7cEXVZNuREtJV6DYAhwSOD/ZuyWSLiAwF8L+lybBNRLJxIve0qr6USltUdRdQjCsi9heRyESvwXiabPDn+wHxWDn7TOCLIrIaKMIVX/+SAjtQ1Q3+txSYgBP/ZD+T9cB6VY2svP4CTvhSkjaSTboKXXdYg+Jl4Eq/fyWuvizi/i3fqjUa2B0oOnQJERHgMWCxqt6bCltE5EAR6e/3e+LqCBfjBO+yNmyI2HYZ8JbPWXQJVb1ZVQ9W1eG45/+Wqn492XaISG8R6RPZB8YCC0hy+lDVzcA6ETnWO30Ot25y0tNpSkh1JWGiNlyr0TJc/dAtCY5rPLAJqMN9Oa/B1e9MBZYDU4CB3q8AD3q75gOj4mjHWbiixzxgjt8uSKYtwCeB2d6GBcCvvfsRwIfACuB5oId3z/XHK/z5IxLwfMbQ3OqaVDt8fHP9tjCSFlOUPkYCM/2z+TcwIBV2pGKzkRGGYaQ96Vp0NQzDaMKEzjCMtMeEzjCMtMeEzjCMtMeEzjCMtMeEzkgoIvJjEemVajuMcGPdS4yE4kcmjFLV7rDKlRFSLEdnxA0/CuBVPxfdAhH5DXAQUCwixd7PWBF5X0RmicjzflxuZM62P/p52z4UkaNSeS9GemFCZ8ST84CNqnqSqp6Imz1kI1CoqoUiMgj4FXCOukHuM3HzxUXYraojgAd8WMOICyZ0RjyZD5wrIn8QkU+r6u6o86NxE4G+66dxuhI4LHB+fOD3jIRba4SGrI69GEZsqOoyP+X2BcCdIjI1yosAk1X18rYu0ca+YXQJy9EZcUNEDgIqVfX/gHtw0wCV46Z1B/gAODNS/+br9I4JXOKrgd/3k2O1EQYsR2fEkxHAPSLSiJvJ5Xu4IujrIrLR19NdBYwXkR4+zK9ws8wADBCReUAN0FauzzA6jXUvMboF1g3FSCRWdDUMI+2xHJ1hGGmP5egMw0h7TOgMw0h7TOgMw0h7TOgMw0h7TOgMw0h7TOgMw0h7/j9zZTE0jeNERwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learning_rate = 0.03\n",
    "iterations = 600\n",
    "initial_w = np.random.randn(train_X1.shape[1])\n",
    "w_sto, weights_history, cost_history = rl_functions.stochastic_gradient_descent(train_X1,\n",
    "                                                                                train_y,\n",
    "                                                                                initial_w,\n",
    "                                                                                learning_rate,\n",
    "                                                                                iterations,\n",
    "                                                                                32)\n",
    "plots.simple_step_plot([cost_history],\n",
    "             \"loss\",\n",
    "             'Training loss\\nlearning rate = {} | iterations = {}'.format(learning_rate,\n",
    "                                                                          iterations))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R² score = 0.6913\n"
     ]
    }
   ],
   "source": [
    "prediction = rl_functions.linear_regression_prediction(test_X1, w_sto)\n",
    "r_2 = util.r_squared(test_y, prediction)\n",
    "print(\"R² score = {:.4f}\".format(r_2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sanity check with sklearn SGDRegressor model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean error between sklearn SGD regressor and my linear regression with SGD: 0.0032  Variance: 0.1170\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import SGDRegressor\n",
    "lr = SGDRegressor(max_iter=1000, loss=\"squared_loss\")\n",
    "lr.fit(train_X, train_y.ravel())\n",
    "pred = lr.predict(test_X)\n",
    "print('Mean error between sklearn SGD regressor and my linear regression with SGD: {:.4f}  Variance: {:.4f}'.format(abs(np.mean(pred-prediction)), np.std(pred-prediction)))\n",
    "print('----------')\n",
    "#print(pred - prediction)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Example of hyperparameter search for batch GD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best hyperparameters\n",
      "learning rate = 0.02 | iterations = 600\n",
      "w = [ 5.87975000e+00 -1.01309419e-01 -1.90966148e-01 -9.27382774e-03\n",
      " -1.54572784e-03 -2.44015931e-02  1.13577394e-01 -4.32606249e-02\n",
      "  1.77223090e-01  1.09100199e-04  5.16773251e-02  5.57692379e-01]\n",
      "lowest validation set cost = 0.5973758219948199\n",
      "\n"
     ]
    }
   ],
   "source": [
    "hyper_params = [(0.001, 200),\n",
    "                (0.1, 10),\n",
    "                (0.9, 8),\n",
    "                (0.02, 600)]\n",
    "\n",
    "all_costs = []\n",
    "all_w = []\n",
    "\n",
    "for param in hyper_params:\n",
    "    learning_rate = param[0]\n",
    "    iterations = param[1]\n",
    "    w, weights_history, cost_history = rl_functions.batch_gradient_descent(train_X1,\n",
    "                                                                           train_y,\n",
    "                                                                           initial_w,\n",
    "                                                                           learning_rate,\n",
    "                                                                           iterations)\n",
    "    all_costs.append(rl_functions.compute_cost(valid_X1, valid_y, w))\n",
    "    all_w.append(w)\n",
    "    \n",
    "best_result_i = np.argmin(all_costs)\n",
    "best_w = all_w[best_result_i]\n",
    "lowest_cost = all_costs[best_result_i]\n",
    "best_params = hyper_params[best_result_i]\n",
    "\n",
    "result_str = \"Best hyperparameters\\n\"\n",
    "result_str += \"learning rate = {}\".format(best_params[0])\n",
    "result_str += \" | iterations = {}\\n\".format(best_params[1])\n",
    "result_str += \"w = {}\\n\".format(best_w.flatten())\n",
    "result_str += \"lowest validation set cost = {}\\n\".format(lowest_cost)\n",
    "\n",
    "print(result_str)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
	#!/bin/bash
	wget https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-white.csv