Exercise 11 Support Vector Machine.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "from scipy.optimize import minimize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAEACAYAAABBDJb9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEGRJREFUeJzt3WGIZeV9x/HfbzurrHH0TcAUlxjbYJudadSlMVPWXU8U\nWzGNppCCpSXUgm9qUEgbtErx9o20oSVIm0JDUqlBEWIrURvLbmtOZk1Ga6KbODMapZBkbeMSi3Sv\nLIjj/vvi3NmM48ydc/c8c8+9z/1+YLjnXp/7nP9ed3773Oc85xxHhAAA421H2wUAAJojzAEgA4Q5\nAGSAMAeADBDmAJABwhwAMpAkzG2fa/trtl+wvWT7oyn6BQDUM5Won3skfSMiftf2lKSzEvULAKjB\nTU8asn2OpOci4pfTlAQAGFSKaZYLJb1m+17bz9r+ku1dCfoFANSUIsynJO2V9MWI2CvphKTbE/QL\nAKgpxZz5K5KORsR3e88fknTb+ka2uQgMAJyGiPBWbRqPzCPimKSjti/qvXSVpOVN2o78z1133dV6\nDdSZ6Of4ccXFFyumpqrH48dHr8Zx+Syps7WfulKtM79F0v22j0i6WNLdifoFTt/iorS0JK2sSMvL\n1TaQqSRLEyPi+5I+kqIvIJnZWWlmpgryPXuqbSBTqdaZZ6MoirZLqIU6a5ielg4frkbkMzPV8w3w\nWaZFne1ovM689o7sGNa+ACAXthXDOAAKAGgfYQ4AGSDMASADhDkAZIAwB4AMEOYAkAHCHAAyQJgD\nQAYIcwDIAGEOABkgzAEgA4Q5AGSAMAeADBDmAJABwhwAMkCYA0AGCHMAyECS28bZ/pGk/5N0UtJb\nEXFZin4BAPWkugfoSUlFRLyeqD8AwABSTbM4YV9AM92utLBQPQITIlUAh6RDtp+xfVOiPoHBdbvS\n/v3SgQPVI4GOCZEqzPdFxF5J10q62fblifoFBrO4KC0tSSsr0vJytY1349tLdpLMmUfET3uPP7P9\nsKTLJD25vl2n0zm1XRSFiqJIsXvg52ZnpZmZKsj37Km28U6r316WlqrP5/BhaXq67arQU5alyrIc\n+H2OiEY7tn2WpB0R8Ybt90g6KOkvIuLgunbRdF9ALd3uz4OKkHq3hYVqGmplRdq5U5qfl+bm2q4K\nm7CtiPCW7RKE+YWSHlY1bz4l6f6I+MsN2hHmwChYHZmvfnthZD7ShhbmdRHmwAjh28vYIMwBIAN1\nw5y14QCQAcIcADJAmANABghzAMgAYQ4AGSDMASADhDkAZIAwB4AMEOYAkAHCHAAyQJgDQAYIcwDI\nAGEOABkgzAEgA4Q5AGSAMAeADBDmAJABwhwAMpAszG3vsP2s7UdS9QkAqCflyPxWScsJ+wMA1JQk\nzG3vlnStpC+n6A8AMJhUI/MvSPqcpEjUHwBgAFNNO7D9cUnHIuKI7UKSN2vb6XRObRdFoaIomu4e\nALJSlqXKshz4fY5oNpi2fbekP5C0ImmXpGlJ/xIRn17XLpruCwAmjW1FxKaD5FPtUgas7Ssk/UlE\nXLfBfyPMAWBAdcOcdeYAkIGkI/O+O2JkDgADY2QOABOEMAeADBDmAJABwhwAMkCYA0AGCHMAyABh\nDgAZIMwBIAOEOQBkgDAHgAwQ5gCQAcIcADJAmANABghzAMgAYQ6k1O1KCwvV43a0BzZBmAOpdLvS\n/v3SgQPV41YBPWh7oA/CHEhlcVFaWpJWVqTl5Wo7ZXugD8IcSGV2VpqZkXbulPbsqbZTtgf64LZx\nQErdbjXCnpmRpqfTt8fEqXvbuMZhbvtMSfOSzuj9fD0i7tigHWEOAAOqG+ZTTXcUEW/a/lhEnLD9\nC5K+bXtfRHy7ad8AgHqSzJlHxIne5pm9Pl9P0S+ANVjGiD6ShLntHbafk/SqpDIillP0C2TndAOZ\nZYzYQuNpFkmKiJOSLrV9jqSDtq+IiG+tb9fpdE5tF0WhoihS7B4YD6uBvHrA8/Dh+gc9N1rGODe3\nvfWiFWVZqizLgd+XfDWL7T+XdCIi/mbd6xwAxWRbWKhG1isr1XLE+fn6gbz6D8HycrWMcZB/CNrW\n7Vb/GM3Ojk/NI6TuAdDG0yy232v73N72LklXSzrStF8gO03WlU9PVwE+Pz9+Qc700FCkWJr4a5L+\nSZJV/ePw1Yj46w3aMTIHJm1deZNvI5A0xHXmdRHmwAQa5+mhEUGYAxgNk/ZtJDHCHAAyMLQDoACA\n9hHmAJABwhwAMkCYA0AGCHMAyABhDgAZIMwBIAOEOQBkgDAHgAwQ5gCQAcIcADJAmANABghzAMgA\nYQ4AGSDMASADhDkAZIAwB4AMNA5z27ttP2F7yfbztm9JURgAoL7Gt42z/T5J74uII7bPlvQ9SddH\nxIvr2nHbOABYq9uVFhel2dlN7486tNvGRcSrEXGkt/2GpBcknd+0XwDIWrcr7d8vHThQPXa7jbpL\nOmdu+wOSLpH0dMp+ASA7i4vS0pK0siItL1fbDUwlKku9KZaHJN3aG6G/S6fTObVdFIWKoki1ewAY\nL7Oz0sxMFeR79lTbksqyVFmWA3fXeM5ckmxPSXpM0uMRcc8mbZgzB4C1ut1qRD4z03jOPFWY3yfp\ntYj4bJ82hDkADGhoYW57n6R5Sc9Lit7PHRHxb+vaEeYAMKChjszrIMwBYHBDW5oIAGgfYQ4AGSDM\nASADhDkAZIAwB4AMEOYAkAHCHAAyQJgDQAYIcwDIAGEOABkgzAEgA4Q5AGSAMAeADBDmAJABwhwA\nMkCYAzg93a60sND4rvJIgzAHMLhuV9q/XzpwoHok0FtHmAMY3OJidSPilZXq7vJLS21XNPEIcwCD\nm52t7ii/c6e0Z0+1jVYluQeo7a9I+m1JxyLiw5u04R6gQE663WpEPjMjTU+3XU22hnpDZ9uXS3pD\n0n2EOQCkM9QbOkfEk5JeT9EXAGBwzJkDQAamhrmzTqdzarsoChVFMczdA8DIK8tSZVkO/L4kc+aS\nZPsCSY8yZw4A6Qx1znx1n70fAMCQJQlz2w9I+o6ki2z/xPaNKfoFANSTbJplyx0xzQIAA2tjmgUA\n0BLCHAAyQJgDQAYIcwDIAGG+jbh2P4BhIcy3CdfuBzBMhPk24dr9AIaJMN8mXLsfwDBx0tA24tr9\nAJoa6s0p6pjEMAeApjgDFAAmCGEOABkgzAEgA4Q5AGSAMAeADBDmAJABwjwTXAcGmGyEeQa4DgwA\nwjwDXAcGQKobOl9j+0XbL9m+LUWfqI/rwABofDq/7R2SXpJ0laT/kfSMpBsi4sV17TidfxtxHRgg\nT8M8nf8ySS9HxI8j4i1JD0q6PkG/GMD0tDQ3R5ADkypFmJ8v6eia56/0XgMADMnUMHfW6XRObRdF\noaIohrl7ABh5ZVmqLMuB35diznxOUiciruk9v11SRMRfrWvHnDnGVrdbrRqanWUqC8M1zDnzZyR9\n0PYFts+QdIOkRxL0C4wE1vFjHDQO84h4W9JnJB2UtCTpwYh4oWm/wKhgHT/GAXcaArawOjJfXq7W\n8R8+zFQLhofbxgEJsY4fbSHMASAD3AMUACYIYQ4AGSDMW8L1xwGkRJi3gHXLAFIjzFvAumUAqRHm\nLUh1/XGmagCsYmliS5quW16dqlntgxNZgDyxzjxzCwvVnPvKSjXCn5+vrmcOIC+sM88ct4oDsBYj\n8zHGKeZA/phmAYAMMM0CABOEMG8RSwsBpEKYt4SzQAGkRJi3hLNAAaREmLeEpYUAUmq0msX2pyR1\nJH1I0kci4tk+bVnNsg5LCwFsZShLE23/iqSTkv5B0p8S5gCQVt0wn2qyk4j4YW9nW+4IALB9mDMH\ngAxsOTK3fUjSeWtfkhSS7oyIR7erMABAfVuGeURcnWpnnU7n1HZRFCqKIlXXAJCFsixVluXA70ty\nbRbb31R1APR7fdpwABQABjSUa7PY/qTto5LmJD1m+/Em/QEATg9XTQSAEcZVEwFgghDmAJABwhwA\nMkCYA0AGCHNMDG4GgpwR5pgI3AwEuZvIMGeENnm4GQhyN3FhzghtMnEzEORu4k4aWliognxlpfrF\nnp+X5ubargrDwM1AMI6GcnOKQYxKmK+OzJeXqxHa4cP8YgMYXYR5H4zQAIwLwhwAMsC1WYaI1TEA\n2kaYN8TqGACjgDBviPXLAEYBYd7QpK1fZkoJGE0cAE1gUlbHrE4prf5ZWdYJbD9WsyA5TrgCho/V\nLEhu0qaUgHHSaGRu+/OSPiHpTUn/JenGiDi+SVtG5hmYlCklYFQMa2R+UNJMRFwi6WVJf9awv9aV\nZdl2CbW0Vef0dDW1UjfIx+HzHIcaJepMbVzqrKtRmEfEv0fEyd7TpyTtbl5Su8blfzB1pjMONUrU\nmdq41FlXyjnzP5L0eML+AAA1TW3VwPYhSeetfUlSSLozIh7ttblT0lsR8cC2VAkA6Kvx0kTbfyjp\nJklXRsSbfdpx9BMATkOdA6Bbjsz7sX2NpM9JOtAvyOsWAwA4PU2XJr4s6QxJ/9t76amI+OMUhQEA\n6hvaGaAAgO0z1DNAbX/e9gu2j9j+Z9vnDHP/ddn+lO1F22/b3tt2PWvZvsb2i7Zfsn1b2/VsxPZX\nbB+z/YO2a+nH9m7bT9hesv287Vvarmkjts+0/bTt53q13t12TZuxvcP2s7YfabuWfmz/yPb3e5/p\nf7Zdz0Zsn2v7a73MXLL90X7th306/7icZPS8pN+R9K22C1nL9g5JfyfptyTNSPo927/ablUbuldV\njaNuRdJnI2JG0m9IunkUP8/e8aiPRcSlkj4s6Urb+1ouazO3Slpuu4gaTkoqIuLSiLis7WI2cY+k\nb0TEhyRdLOmFfo2HGubjcpJRRPwwIl5WtQxzlFwm6eWI+HFEvCXpQUnXt1zTu0TEk5Jeb7uOrUTE\nqxFxpLf9hqpflvPbrWpjEXGit3mmqt/bkft8be+WdK2kL7ddSw3WCF+bqjdrsT8i7pWkiFjZ7FIp\nq9r8w3CS0eDOl3R0zfNXNKLhM25sf0DSJZKebreSjfWmL56T9KqkMiJGcfT7BVWr28bhQFxIOmT7\nGds3tV3MBi6U9Jrte3vTVl+yvavfG5KHue1Dtn+w5uf53uMn1rRp/SSjOnViMtg+W9JDkm7tjdBH\nTkSc7E2z7JZ0wPYVbde0lu2PSzrW+6Zjjd632vX2RcReVd8kbrZ9edsFrTMlaa+kL/bqPCHp9q3e\nkFREXN3vv/dOMrpW0pWp9z2IreocUf8t6f1rnu/uvYbTZHtKVZB/NSK+3nY9W4mI47b/VdKva7SO\n6eyTdJ3tayXtkjRt+76I+HTLdW0oIn7ae/yZ7YdVTWE+2W5V7/CKpKMR8d3e84ck9V3wMOzVLKsn\nGV231UlGI2SURhjPSPqg7QtsnyHpBkmjumpgHEZnkvSPkpYj4p62C9mM7ffaPre3vUvS1ZKOtFvV\nO0XEHRHx/oj4JVV/L58Y1SC3fVbv25hsv0fSb0pabLeqd4qIY5KO2r6o99JV2uLA8rDnzP9W0tmq\n5qqetf33Q95/LbY/afuopDlJj9keibn9iHhb0mdUrQpakvRgRPQ9wt0G2w9I+o6ki2z/xPaNbde0\nkd6KkN9XtTrkud7fyWvarmsDvyjpm70586ckPRIR/9FyTePsPElPrvk8H42Igy3XtJFbJN1v+4iq\n1Sx9l6Ry0hAAZGBkl+YAAOojzAEgA4Q5AGSAMAeADBDmAJABwhwAMkCYA0AGCHMAyMD/A+Rrj9vR\nmeOAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1068d1fd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "np.random.seed(9999)\n",
    "xp = np.random.randn(10, 2)\n",
    "xm = np.random.randn(10, 2) + [3, 3]\n",
    "plt.plot(xp[:, 0], xp[:,1], '.b')\n",
    "plt.plot(xm[:, 0], xm[:,1], '.r')\n",
    "data = np.concatenate([xp, xm], axis=0)\n",
    "classes = np.array([1]*10+[-1]*10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1) What is an equation of line/plane? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2) Explain geometrical interpretation of the two parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3) How do you determine whether a point $\\vec{x}$ is one side of the line or another?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "4) Given a line, is $\\vec{w}$ and $b$ unique. (Are there two equation that represents the same line?)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5) Give another equation for the line represent by $\\vec{w}=[1,2]$, $b=3$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "6) If we want $b$ in the previous equation to be 1. What should $\\vec{w}$ be"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#minimize_with(w,b) 1/2 |w|^2\n",
    "#subject to y*(w dot x + b) = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "7) Draw a line defined by $\\vec{w}=[2,3]$, $b=1$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "8) Draw a green point on the side where $\\vec{w}\\cdot\\x + b > 0$ and a blue point on the other side."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "9) What is a linearly separable data?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "10) What is a margin?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "11) How do we calculate the margin? And with that formula what kind of normalization are we using?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "12) What are we trying to optimize? What are our contraint?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "13) Can we use gradient descent with constrained optimization? Why?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "14) Given these data points find the line of maximum margin. Draw it along with data points.\n",
    "\n",
    "Use numpy.optimize.minimize with method = SLSQP\n",
    "\n",
    "Read the doc here.\n",
    "http://docs.scipy.org/doc/scipy/reference/tutorial/optimize.html#constrained-minimization-of-multivariate-scalar-functions-minimize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  status: 0\n",
       " success: True\n",
       "    njev: 2\n",
       "    nfev: 8\n",
       "     fun: 19.999999999988923\n",
       "       x: array([ 5.,  4.])\n",
       " message: 'Optimization terminated successfully.'\n",
       "     jac: array([ 8.,  4.,  0.])\n",
       "     nit: 2"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Example\n",
    "\n",
    "def f(x):\n",
    "    a, b = x\n",
    "    return (a-1)**2 + (b-2)**2\n",
    "\n",
    "# minimize f(x)\n",
    "# subject to: x -----> (a, b)\n",
    "# a - 5 >= 0\n",
    "# b - 4 >= 0\n",
    "\n",
    "def constraints(x):\n",
    "    a, b = x\n",
    "    return [a-5, b-4]\n",
    "\n",
    "cons = ({'type': 'ineq',\n",
    "         'fun' : constraints})\n",
    "\n",
    "minimize(f, [10., 10.], method=\"SLSQP\", constraints=cons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  status: 0\n",
      " success: True\n",
      "    njev: 5\n",
      "    nfev: 25\n",
      "     fun: 0.88345210488050663\n",
      "       x: array([ 2.55905484, -0.7316033 , -0.59009212])\n",
      " message: 'Optimization terminated successfully.'\n",
      "     jac: array([ 0.        , -1.46320659, -1.18018422,  0.        ])\n",
      "     nit: 5\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x106b36a90>]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAEACAYAAABBDJb9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt0lFWe7//3BsJdEARBRUBAQS7hVrGhRYkiF6VFFJD7\nHULq6T59ZvqsWWd65qxF85s1s/rMObNmes7keQIiV0EUFURBRcF4F1PcCVcFFEFAQEi457J/f3wT\ngnSAYCr1VD35vtbKagqepL5Fy7d27Wd/9jbWWpRSSiW2an4XoJRSquK0mSulVABoM1dKqQDQZq6U\nUgGgzVwppQJAm7lSSgVAuZu5MeZFY8wxY8y2q36vkTFmrTFmjzHmPWNMw8opUyml1I3cysh8PjDw\nmt/7e+ADa217YD3wx2gVppRSqvzMrYSGjDGtgLestcnFj3cDfa21x4wxzYEsa22HyilVKaXU9VR0\nzvxOa+0xAGvtUeDOipeklFLqVkX7BqjuDaCUUj6oUcHvP2aMaXbVNMvx611ojNFGr5RSv4C11tzs\nmlsdmZvirxKrgEnFv54IvHmTguL+a+bMmb7XoHVqjVqn1lnyVV63sjRxKfA58IAx5jtjzGTgz0B/\nY8weoF/xY6WUUjFW7mkWa+2Y6/zRE1GqRSml1C+kCdBrpKam+l1CuWid0ZMINYLWGW2JUmd53dI6\n8wo9kTE2Vs+llFJBYYzBVsINUKWUUnFIm7lSSgWANnOllAoAbeZKKRUA2syVUioAtJkrpVQAaDNX\nSqkA0GaulFIBoM1cKaUCQJu5UkoFgDZzpZQKAG3mSikVANrMlVIqALSZK6VUAMS2mf/xj3DwYEyf\nUimlqoLYNvPLlyEUgt/8BlavhsLCmD69UkoFVewPpzh/HpYtA8+DEydgxgyYOhWaNo1JHUoplUhi\nejiFMeaPxpgcY8w2Y8wSY0zN615cty5MmQLZ2fDqq7BnD9x/P4wbB59/DnoakVJK3bIKj8yNMa2A\nD4EO1trLxphXgNXW2kXXXHf9Y+NOnYIFC2S0Xq8eOA6MGQP161eoNqWUSnSxHJnnApeBesaYGkBd\n4Mgt/YTGjeEPf5BR+r/+K6xZAy1bwu9/D7t2RaFEpZQKtgo3c2vtT8C/Ad8Bh4HT1toPflk11WDA\nAFi5ErZsgQYN4PHH4bHHYPlyyM+vaLlKKRVI0ZhmaQO8DfQBzgCvAcuttUuvue760yw3cvkyrFgB\nrgv79sH06fLVokWF6lZKqURQ3mmWGlF4rhDwmbX2VPETvwH8Glh67YV/+tOfrvw6NTWV1NTUm//0\nmjVh5Ej5ysmRefXkZEhNlbn1xx+XEb1SSgVAVlYWWVlZt/x90RiZdwVeAlKAS8B8INtam3HNdb9s\nZF6WvDxYskRG65cuQXo6TJoEjRpF5+crpVSciNkNUGvtVmARsBHYChhgTkV/7g3ddps08K1bYd48\nWeZ4332yXn3jxkp9aqWUikexDw1VluPH4cUXITMTmjeXKZjnn4c6dSrvOZVSqpKVd2QenGZeorAQ\n3nlHpmCys2HiRBnFt2tX+c+tlFJRFtMEaFypXl32flmzBr78EoyB3r1h0CBYtUr3g1FKBVLwRuZl\nuXhR1qm7Lhw5UrofTLNm/tSjlFLlVHVH5mWpXRvGj4cvvpBA0sGD0KEDjB4Nn3yi+8EopRJe1RiZ\nl+X0aVi0SEbrSUkQDstmXw0a+F2ZUkpdUXVvgN4qa+HDD6Wpr18Po0ZJY+/Sxe/KlFJKp1nKzRhJ\nkb72GmzfLvPogwbBI4/Ayy/LdgJKKRXndGRelvx8WfniurKFwNSpctO0ZUu/K1NKVTE6Mq+IpCQY\nNgzWrYOsLDh7Frp3h2eegffeg6IivytUSqmf0ZF5eZ07J9Murgu5uRJEmjwZ7rjD78qUUgGmI/No\nq1cPpk2TvV+WLJH59bZtJWG6YYMub1RK+SqmI/OCAkv16jF5utg4cQLmz5f9YG6/XVbBjB4tjV8p\npaIgLkfmbdrAv/wLHDsWy2etRE2awN/9nRya8c//DG++KTdJ/+Zv5Ag8pZSKkZg28xUr4MABCV+O\nGROg8GW1arKc8a23ZBqmbl149FF44gl44w0oKPC7QqVUwPlyA/T0aVi4UA4NKglfjh8v25QHxqVL\n8Prr8iIPHIC0NJlzv/tuvytTSiWQhEiAWiuhS88rDV86DnTuHJOSYmf7dlkF88or0K+fvMjUVAks\nKaXUDSREM7/a4cPwwgvy1bat9LvnnpMjQAMjNxdeekkae1GRLG+cMEFuniqlVBkSrpmXyM+X+4iu\nCzt3ysxEWlrAwpfWyg0Dz4N334Xhw+Xdq3t3vytTSsWZuFzNUh5JSdLb1q+X/a/y8gIYvjRGbpC+\n/DLs2gWtW8sL7N0bFi+W/deVUuoWRGVkboxpCMwFOgNFwBRr7YZrrvnFCdBz52DpUhmt5+XJDdNJ\nkwIWviwogNWrISMDtmyRdOmMGbKeUylVZcV6ZP4XYI219kGgK7ArSj8XkAzO9OmwaZNMOW/bJkd6\nTpoEX30VkOWNNWrI6HztWvjsM2nuDz0EgwdLk9fj7pRSN1DhkbkxpgGw2Vrb9ibXRXVvlpLwpedB\no0Yy5Tx6tCzxDowLF2QFjOvC8eNyw3TqVGja1O/KlFIxEsuR+X3ACWPMfGPMJmPMHGNMnSj83Bsq\nCV9+/TX80z/JaXAtW8Lf/i3s3VvZzx4jdeqUfvx47TVJmj7wgJyI9PnnAflIUsny8uS4wLw8vytR\nqlJFY2TeE/gS6G2tjRhj/gM4Y62dec11dubM0t9KTU0lNTW1Qs99rYMHYfZsmDcPkpNlbn3IEJnB\nCIyffpLEletKs3ccGDsW6tf3u7L4k5cnh4zk5ECnTrKCKFDJNBVEWVlZZGVlXXk8a9as2CxNNMY0\nA76w1rYpftwH+J/W2qevuS5mW+CWhC9dVxp8WprMud91V0yePjaKimTJj+vKnutjxsi7V6dOflcW\nP774QlYNFRTIMqmPP4ZevfyuSqlbErNpFmvtMeCQMeaB4t/qB+ys6M+tiFq1pLd9+imsWQM//AAd\nO8Lzz8tyx0DMTlSrVrr3y9at0LixPE5NhVdf1ePuQKLEnTpJI+/YUd/oVKBFa2liV2RpYhKwH5hs\nrT1zzTW+Hk6RmytLuD1PBrXhsIQvGzb0raTou3xZbh64ruzaOH26fCxp0cLvyvyTl1c6zaJTLCoB\nJWwCtLKVhC8zMmQV4IgRMu3crZvflUVZTo7ss75kCfTtKy+yXz8Z0SulEoY283I4ehTmzpWbpvfe\nK/1u+HCoXdvvyqLo7Flp6K4rSx1LEleNGvldmVKqHLSZ34KCAnj7bel3JeHL9HS47z6/K4sia+WG\noOfJi33uOXn36tnT78qUUjeQsHuz+KFGDRg6tDR8mZ8PKSkBC18aA7/+tdw42LtX1qsPHy4p0wUL\nZNSulEpYOjK/jvPnJXzpefDjjzJSnzIlYOHLwkLZtdF15VDqiRPlhd5/v9+VKaWK6ci8gurWlemW\nr76C5ctLB7PjxwcofFm9eunHj6++ko8oDz8MAwbIqhg97k6phKEj81tw6pTMSHiebP4VDgcwfHnx\norx7eR4cOiQ7N06bBs2b+12ZUlWS3gCtREVFsG6dLG/8+GNp6OGw5FICZcsWaeqvvgoDB8oN00ce\n0ePulIqhuJxm+ejgRwShoVerBv37y0zE1q1y6lu/fgEMX3brJus2Dx6EPn1kPr1LF3kXy831u7rr\n0821VBUU05F5xwwZuoZDYcYnj6dh7eDELy9fhhUrZCAb2PCltbIPjOfB++/DyJEyWk9O9ruyUrq5\nVvnk5cGOHbLlgf79xLW4HJnvCO/Afcrl428/pvVfWjPjrRlsObolliVUmpo1pbdlZcEHH8j8enKy\nLOf+4IMAHXf32GPy8SMnB+6+G556SkbtS5fKDmd+27FDaisokENkc3L8rij+lLzhPfqo/K9+ggkE\n3+bMf8j7gRc3v8jsjbNp2bAl4VCY4R2HU7tGcOKXeXmlx91dvCjz6hMnBix8mZ8Pb70lL3L7dlm/\nOWOGnGvqh5JGtXOn3MTQkflf090kE0rC3AAtKCrg7b1v42a7bDm6hSndpzCj5wzuaxSc+KW1spzR\n82QVYGDDl3v2yH4wixZJQMlx5MZprPeD0c21bkzf8BJKwjTzq+09uZfMSCaLti6iV4teOCkOA9sO\npHq16jGpMRaOH4cXX5T7infeKf1u5Eg5ZyIwzp2DZctktH76tNw4nTxZjodS8UHf8BJGQjbzEufz\nz7NsxzLcbJdTF06RHkpnSvcpNKkbnGZQWAjvvCP9Lju7NHzZrp3flUWRtfLiXBfefFOOfQqH4Ve/\n0uWNSpVTQjfzq2UfzsaNuKzcvZLfPPAbwqEwvVv0xgSoGezfLyP1+fOhe3cZrQ8eHLDj7k6eLE1c\nNWhQegJ3vXp+V6ZUXAtMMy9x6sIpFmxZgBfxqJdUDyfFYUyXMdSvGZz45cWLslDE8+Dw4dLwZbNm\nflcWRUVFsqOZ58lRUOPGyWi9Qwe/K1MqLgWumZcoskWs27+OjOwMPv72Y8Z2GUs4JUzHpsGKX27a\nJP3utddg0CAZyPbpE7DZiW+/hTlz5CZCx47yIp95RlZYKKWAADfzqx06c4g5G+cwd/NcOjTpgBNy\nGNphKEnVg9MMTp+WxSGeJ/tiOY4MZhs08LuyKLp8Wc4ydV345htJXE2fDvfc43dlSvmuSjTzEpcL\nL7Ni1wrciMu+k/uY3mM603tOp0WD4MQvS8KXriv7wowaJbMTXbr4XVmUbd8u71wvvyx7JDiOBJUC\n9ZFEqfKLeTM3xlQDIsD31tohZfx5TDba2nF8B5mRTJZuX0pq61ScFId+9/UL1A3TI0fghRfk6777\npKkPGwa1avldWRTl5ZWewF1QUHoC9+23+12ZUjHlRzP/W6An0MDPZl4i71IeS7Yvwc12uVR4iXAo\nzMSuE2lUJzjxy5LwZUaGpNinTpWbpq1a+V1ZFFkrN0o9T9ZyDh8uo/Xu3f2uTKmYiOneLMaYFsBT\nwNxo/LxouK3WbaSH0tmavpV5Q+aRfSSbNv/ZhqlvTmXjkY1+lxcVSUmSJl23Dj76SE5H6tFDlnO/\n+26A9oN55BHZF2H3bvko8swz0Lu33Ey4eNHvCpWKC1EZmRtjlgP/DDQE/kc8jMzLcvzcceZtnkdm\nJJNm9ZsRDoUZ2WkkdZKCE788d06mm10XzpwpPe7ujjv8riyKCgtlXwTPg40bSxNXbdv6XZlSURez\naRZjzGDgSWvt74wxqUgzf7qM6+zMmTOvPE5NTSU1NbVCz/1LFRYVsmbfGryIR/aRbCZ2nUh6KJ12\njYMTv7RWToJzXVi1SkbrjiPnNwfo9gF8/bXsB7NwIYRC8iKfekqW/iiVgLKyssjKyrryeNasWTFr\n5v8CjAMKgDrAbcAb1toJ11zn+8i8LN+c+obZG2czf8t8et7VEyfFYfD9gwO1H8zJk5Iu9Ty5fxgO\nBzB8eeGCJK5cF44dk5sHU6fKBjhKJTBfliYaY/oSx9MsN3Kx4CLLc5bjRlwO5x5mRs8ZTOsxjWb1\ngxO/LCqSMyVcV+4pjh8vsxOBC19u3CjvXK+/Dk8+KaP1hx8O2EcSVVXE5eEU8ax2jdqM7zqeL6Z+\nwcpRKzl4+iAdMjow+vXRfPLtJ4E57m7gQNnzatMmqFsX+vaV5dyvvy6rYwKhZ0+YO1c2vXnoIRmh\nd+0q0zF6EIMKqECEhirL6YunWbR1EW62S1L1JJyQw7jkcdxWKzhbhl66JOHLjAw4cECOups+XQ4R\nCgxrYf16+Ujy4YcyxxQOy5FpSsW5KpUArWzWWrIOZuFGXNbtX8eozqMIh8J0aRas+OW2bTI7sWxZ\ngMOXhw9L2mrOHNlv2HFkfWfNmn5XplSZtJlXksO5h5m7aS5zNs2hTaM2OCGH5x58jlo1ghO/zM2F\nl16SgWxhYUDDl/n5sHKlvHvt3CnbU6alQcuWflem1M9oM69k+YX5vLnnTbyIR87xHKZ2n0pazzRa\n3R6c+GVJ+NJ1JYQ0YoQMZLt187uyKNu1S5r6kiUSUHIceOKJ2B93p1QZtJnH0O4Tu8mMZLJ422Ie\nvvdhnBSHAW0HUM0EpxkcOyY71WZmymaGjiPNvXZwzt+Gs2clcZWRIemrkuPuGjf2uzJVhWkz98G5\ny+d4ecfLZGRnkHcpj/RQOpO7TeaOusGJXxYUwJo1MlrfuFF6XXo6tGnjd2VRZK2cYO958PbbMHSo\nvHulpNz8e/PyZKOczp3Ld7bmrV6vqhxt5j6y1rLh8Aa8iMeqPat4pv0zOCkOKXenBGr3xq+/ln63\ncKGsAHQcWdYdqPDljz9K4iozU/ZECIdl/+G6df/62pJT70sOSr7Zqfe3er2qkrSZx4kT508wf/N8\nMjdm0qh2I8KhMKO7jKZuUhnNIEGVhC8zMuD48dL9YAIVviwshPfek48kX34pd4TT0+GBB0qv+eIL\nePRR+fiSlAQffwy9el3/Z97q9apK0mYeZ4psEWu/WYub7fL5oc8Znzye9FA67Zu097u0qIpESsOX\ngwfLaP3Xvw7Y8saDB+UE7nnzIDlZXuTTT8u72iOPyOqYjh3LHmlfPa0CN7/+et+rI/gqQ5t5HDt4\n+iBzNs7hxc0v0uXOLjgpDkPaD6FGtRp+lxY1p07JDrWuC3XqSL8bOxbqB+f8bUlcvfaavHt9+60s\nbRw1SjbD6dSp7EZ+7bQKlD4O6pSMvglVSHmbOdbamHzJU6mrXcy/aJdsW2L7zOtj7/63u+2fPvyT\nPZx72O+yoqqw0Nr337f22WetbdzY2t/9ztqcHL+rqgRbtliblmbt7bdbO2KEtR9+aG1R0c+v+fxz\na2vUsBasTUqy9osvyv/zK/K9fsrNtbZrV6m9a1d5rG5Jce+8aY8Nztq5BFSrRi3GdBnDJ5M/4Z2x\n73D07FE6uZ0YsXwEHx74MDD7wTzxhGwZsHWrrPJ74glITYVXXpGznAOha1eZejl4UObBHUdG0P/1\nX7KxPMjItFMnmR/v2FF+XV4V+V4/7dghnyYKCmQ6KSfH74oCS6dZ4kzupVwWb12MF/EoskWEQ2Em\ndJ1Aw9oN/S4takrCl64rhweVhC/vvdfvyqLIWjn+yfNg7Vp4/nlp8G3alG9apSx5eb/8e/1SMj1U\n3vsC6q/onHmCs9byyXef4EU83v36XUZ0HIGT4tCtebDilzt3loYvSwa0gQtf/vCD7OI4Z45sF+A4\ncpZpoE7gvoFEfBOKI9rMA+To2aOyH8zGObRo0IJwKMyITiOoXSM48cuzZ+WYT9eV8GU4DJMmBSx8\nWVAgISTXlTmnKVPkEI3Wrf2uTMUxbeYBVFBUwOq9q3EjLpt/2MzkbpOZEZpBm0bBiV9aK8u4XVf6\n3rPPykA2FPK7sijbu1eCSIsWydpyx5HN5gOVuFLRoM084Pad3MfsjbNZsGUBD93zEE6Kw5PtngzU\ncXcl4UvPgyZNpN+NHFl2+DJhnT8vew67rqznnDFDRuxNm/pdmYoT2syriAv5F1i2YxlexOP4ueOk\nh9KZ2n0qTesFpxmUhC8zMmDDhrLDl4GQnS0vcuVKCSE5jozaA5W4UrdKm3kVlH04Gy/i8cauN/jN\nA7/BSXHo3aJ3oPaDOXCgNHzZrZv0u9/8BmoEJ28loaMFC2Qapn59uYEwZkzAEleqvLSZV2GnLpxi\n4ZaFeBGPOkl1cEIOY5PHUr9mcJpBSfjSdeG772Rp47RpcNddflcWRSUncHueLOkbO1Ya+4MP+l2Z\niqGYNXNjTAtgEdAMKAJesNb+ZxnXaTOPsSJbxLr96/AiHlkHsxjbZSzhlDAdm3b0u7So2rJF+t2r\nr8KAATJaf/TRgM1OHDokH0lefFGaeTgsW/MmJfldmapksWzmzYHm1totxpj6wEbgGWvt7muu02bu\no0NnDvHCpheYu2kuD9zxAE6Kw9AOQ6lZPThnX545I4tDPE8auePA+PHQoIHflUXR5cuwYoV8JNm3\nT07fnj4dWrTwuzJVSXybZjHGrAT+n7V23TW/r808DuQX5rNy90rciMvuE7uZ3mM6aT3TaNEgOM2g\nJHzpujJLMXKkNPbkZL8ri7IdO2RefelS2R/BceDxxwOWuAq4cmxC5kszN8a0BrKAztbas9f8mTbz\nOLPzx5142R5Lti+hb+u+OCGHfm36Beq4u5Lw5ezZks1xHBg2LGDhy7w8OYHb8+RmQjgMEydCo0Z+\nV6ZupJw7Yca8mRdPsWQB/2StfbOMP7czZ8688jg1NZXU1NSoPLeqmLOXz/LStpfwIh4X8i8QDoWZ\n2G0ijesEJ35ZUABvvSWj9W3bAhq+tBY++0xe5DvvyLuW40CPHn5XpspyncNJsrKyyMrKunLZrFmz\nYtfMjTE1gLeBd6y1f7nONToyj3PWWj4/9DluxGX13tU89+BzOCkOobuDFb+sEuHLY8dk/ebs2dCs\nWWniKlAncCe4cm5CFtORuTFmEXDCWvuHG1yjzTyBHD93nHmb5zF742ya1m2Kk+IwstNI6iTV8bu0\nqCkJX2ZkwE8/lR5316SJ35VFUWGhjNJdV0JJkybJC23b1u/KFJRrE7JYrmZ5GPgY2A7Y4q9/sNa+\ne8112swTUGFRIe9+/S5exGPD4Q1M7DqR9FA67Rq387u0qLFW+pznyUKRIUNk2jlw4ctvvpGR+oIF\nMvUSDsvZfoFKXAWPhoZU1B346QCZkUzmb5lP97u644QcBj8wOFDH3ZWELz1PBkqOI+HLevX8riyK\nLl6E5cvlI8mRI3LzYNo0mY5RcUebuao0Fwsu8trO13CzXQ7lHmJGzxlM6zGN5vWb+11a1JSEL10X\nPv0Uxo2TgWyHDn5XFmWbNsk712uvwaBB8u7Vp0/APpIkNm3mKiY2/7AZL+KxfOdyBrQdgBNyeLTV\no4HaD+a77+CFF2SJY8eO0tSfeSZg4cvTp2HhQmnsSUnS1MeN08Mk4oA2cxVTZy6eYdHWRbgRl2qm\nGk7IYXzX8TSoFZz4ZZUIX1oL69fLi/zwQxg9Wt69Onf2u7IqS5u58oW1lqyDWbgRl3X71zGy00jC\nKWGSmwUrfrljhwxiX34ZHnusNHwZoA8kcPiwfCR54QVZ/eI48NxzUDM4W0AkAm3myndH8o5cOe6u\n9e2tcVIchj04jFo1ghO/vDp8eflyafjy9tv9riyK8vPhzTflRebkwNSpctO0UaObRtFVxWkzV3Gj\noKiAt/a8hRtx2XZsG1O7TyWtZxqtb2/td2lRU2XCl7t3lyauCgvl8NbOneUusTb0SqHNXMWlPSf2\n4EU8Fm9bzK/v/TVOyGFgu4GB2g/m2DHZqXb2bNlf3XHg+ecDFr5cvx7695dlPwC/+x386U9wxx2+\nlhVE2sxVXDt3+RzLdiwjIzuD0xdPkx5KZ0r3KTSpG5z4ZWEhrFkjo/VIJGDhy6s3iWrdGlJS5CPJ\nkCHy7vXQQwG7geAfbeYqIVhr+erwV3gRj5W7VzKk/RCcFIdf3fOrQC1vLAlfzp8PPXtKvxs8OMH3\ng7k2in7ihLzAzEy5aeA4shomUCdwx542c5VwTp4/yfwt88mMZNKgVgOcFIfRnUdTr2Zw4pcXLkj4\n0nVle94ZM+R+YqDCl0VFsHatvMjPP5cTQsLhAJ7AHRvazFXCKrJFvP/N+7gRl0+/+5TxyeMJh8K0\nb9Le79Ki6urw5ZNPSr8LXPjy4EGYM0d2cOzcWUbrQ4bofjC3QJu5CoTvznzHnI1zmLtpLp3u7IQT\nchjSfghJ1YMTv/zpJ1kc4rqyhDuQ4ctLl+D11+Xd68ABOYF7+vSAncBdObSZq0C5XHiZ13e+jhfx\n+Oanb5jeYzrTe0znngb3+F1a1FgL69ZJvwt0+HLbNnnneuUVWRETDsuxd4H6SBI92szjQTnO91O3\nbvux7XgRj2U7lvH4fY/jpDg81vqxQN0wrRLhy9xcWLxYGru10tQnTICGDf2uLK5oM/dbOc/3U79c\n3qU8Xtr2Em7EJb8w/8pxd7fXDk78Mj8fVq2SfpeTIzvVpqVBy5Z+VxZF1sq/D9eF996DESPk3atb\nN78riwvazP12nfP9VPRZa/n0u09xIy7vfv0uwx8cjpPi0P2u7n6XFlW7dsmqv5dekhuljiOzFNWC\nk7eSJT4vvig3TVu0kBc5fHjAEle3Rpu538p5vp+KrmNnjzF301xmb5zNPQ3uIRwK83yn56ldIzjN\n4OxZWLpU5tbz8iSINHlywMKXBQWwerWM1jdvLj2B+777/K4s5rSZx4NynO+nKkdhUSGr963GzXbZ\n9MMmJnWbRHoonTaN2vhdWtRYCxs2SL9btQqefVYGsikpflcWZfv2SeJq4UJJljqOHKSR0Imr8tNm\nrlSxr099zezIbBZsXUDK3SmEQ2Geuv8pqlcLTjP48cfS8GXjxtLvRo0KWPjywgVZAeO6cPx46Qnc\nd97pd2WVKqbN3BgzCPgPoBrworX2f5dxjTbzyqQrZ27qQv4FXs15FS/icfTsUdJ6pjG1+1Sa1Q9O\n/LKoSO4huq7ctpkwQXpe4MKXkYi8yBUrZF8Ex4HevQO5vDFmzdwYUw3YC/QDjgDZwChr7e5rrtNm\nXll05cwt23hkI262yxu73+DJdk/ipDg8fO/DgVreePCgzE7MmwfJydLvnn46YOHLU6dk+sV15WOI\n48DYsVC/vt+VRU0sm3kvYKa19snix38P2GtH59rMK5GunPnFfrrwEwu3LsSLeNSqXgsnxWFsl7Hc\nVis4b4Yl4UvXhW+/LT3uLlDhy6IiSVy5Lnz0kTT0cFgWHyS48jbzaCxqugc4dNXj74t/T8VK584y\nIk9Kkv94O3Xyu6KE0ahOI/6m19+w+7e7+feB/87ab9bS6j9a8dvVvyXneI7f5UVFrVowZoycH7F6\ntaz+69hRlnN/+KHcSE141arJOs0VK2DrVjkFqV8/OdNv+XJZsB9w0RiZDwMGWmvTih+PAx6y1v7+\nmuvszJkzrzxOTU0lNTW1Qs+trqIrZ6Lm+9zvr+wHc/8d9+OEHJ5t8QQ1d+0NzD2J3FzZD8bzAhy+\nzM+HlSt27tgdAAATPElEQVQhIwP27i1NXMX5CdxZWVlkZWVdeTxr1qyYTrP8yVo7qPixTrOoQMgv\nzGfl7pV4G/6LXfs+Z2qkkLSzHWi5dkMgGjpII//4Y5mdWLtWTkRyHOja1e/KomznTnnnWroU+vaV\nd69+/RIicRXLOfPqwB7kBugPwFfAaGvtrmuu02auEtMXX7Dr2UfwehSypAv0afUIzoB/pH/b/oE6\n7q4kfDl7tmwXUBK+rBWc87dLE1cZGbLUMRyWI6AaNfK7suvyY2niXyhdmvjnMq7RZq4S01Vp3rNd\n2vPyf6bh7phH3qU8wqEwk7tPpnGdxn5XGTUFBfD22zJa37q1NHzZurXflUWRtbJwICNDzvYrSVyF\nQn5X9lc0NBTvdF14YrnmnoS1li+//xI34vL23rcZ2mEoTsgh5Z5gxS/37ZMg0sKFskDKcWDgwICF\nL48fl/WbmZkSQHIcGDkS6tTxuzJAm3l8i9a6cH1DiAs/nvuR+Vvm40U8mtRtghNyGNl5JHWTghO/\nPH9ewpeeJ2nTkvBl06Z+VxZFhYVyKLXnwVdfwcSJ8kLbtfO1LG3m8Swa68I1KBR3CosKee+b93Cz\nXb78/ksmdJ1AeiidB+4IVvwyO1v6XaDDl/v3l57A3b176QncPiSutJnHs2jsqKhBobh24KcDzN44\nm/lb5tO1WVfCoTBPt3+aGtWCE788dQoWLJDGXq+e3EsMWPgSLl6UQ1ozMuTEkLQ0WeLYvHnMStBm\nHu/y8uSjnLXwq1/dejPXLXYTwqWCS7y+63UysjP47sx3pPVIY1qPadx1W3Dil0VF8MEH0tQ//lgC\nSgEJX/7c5s3yIpcvlxsHjiP/Biv5I4k283gXjWkSDQollK1Ht+JFPF7JeYX+bfrjpDj0bdU3UPvB\nHDok50rMnQsdOki/GzpUPjwGxpkzpSdwV68u71zjx0ODBpXydNrM451Ok1RZZy6eYfG2xbjZLgBO\nisP45PE0rB2c+OXlyzKn7nkSvizZDybOw5e3xlrZD8F1ZV+YUaOksScnR/VptJnHO50mqfKstXz0\n7Ue42S7v73+fkZ1GEg6F6do8WPHLnJzS8GVqqozWH388IcKX5XfkiHwcmTNHFuQ7DgwbFpXElTbz\nRKDTJKrYD3k/XDnurtXtrXBCDsM7DqdWjeDEL/PySsOXly7JIHbixLgOX966ggJ46y0ZrW/bBlOn\nSuKqVatf/CO1mSuVgAqKCnhrz1t4EY+tx7YypdsUZoRm0Pr21n6XFjXWwmefyWh9zRp47jkZyPbs\n6XdlUbZnjwSRFi2Chx+WFzlgwC1/JNFmrlSC23tyL5mRTBZtXUSvFr1wUhwGth0YqOPujh8v3Q8m\nDsOX0XH+PCxbJh9JTp8uPYG7SZNyfbs2c6UC4nz+eV7Z8QoZ2RmcunCKGT1nMKX7FJrWC078siR8\n6boSSoqT8GV0WSvLkT0P3nwThgyRd6+HHrrh8kZt5koFUPbhbNyIy4pdK3i6/dM4IYdeLXoFannj\n1eHLHj1Kw5eB2g/mxInSxFXDhvIiR4+W9NU1tJkrda0A7WVz8vzJK8fd1Uuqh5PiMKbLGOrXDE78\n8uJFyee4roQvZ8yQ8GWz4Jy/LYmrtWvlRX72GYwbJ3eGO3S4cok28xsJ0D9qVU4B3cumyBbxwf4P\nyMjO4JNvP2Fc8jjCoTAPNn3Q79Ki6urw5aBBMpDt0ydg+8F8+21p4qpzZ3mRQ4ZgatbUZl6mgP6j\nVjdRBUJa35357spxdw82fRAn5DC0w1CSqgcnfnn6dOlxd9WrS78bN67Swpf+uHQJ3nhDRuv792OO\nHNFmXqYq8I9alaEKhbQuF15mxa4VuBGXfSf3Mb3HdNJ6pnFPg+Ccs351+HL9+tLwZZcuflcWZdu3\nY5KTtZmXqTL+Ueu0TWKogiGtHcd3kBnJZOn2pTx232OEQ2H63dcvUDdMDx8uDV+2bStN/bnngnPc\nnc6Z30g0/1FXtWkbfeNKSHmX8nhp20t4EY9LhZcIh8JM7DqRRnWCE7/Mz4dVq2S0npNTGr5s2dLv\nyipGm3msVKVpm6r2xhVA1lo+O/QZbrbLmn1rGPbgMJwUh553Byt+uXu3hC8XL5YbpY4D/fsn5n4w\nMWnmxph/BZ4GLgHfAJOttbnXuTaYzbwKzcVWqTeuKuDY2WPM2zyPzI2ZNK/fHCfk8Hyn56mTFJz4\n5blz8PLLMlrPzS0NX95xh9+VlV+smvkTwHprbZEx5s+Atdb+8TrXBrOZQ9WZi61Kb1xVSGFRIWv2\nrcGNuESORJjUdRLpoXTaNm7rd2lRUxK+zMiQfbDKGb6MCzGfZjHGDAWGWWvHX+fPg9vMq5Kq8sZV\nRX1z6htmb5zNgi0L6Hl3T8KhMIPvHxyo/WBOnJB0aWbmTcOXccGPZr4KWGatXXqdP9dmrlSCuFhw\nkeU5y3EjLkfyjjCj5wymdp9Ks/rBiV8WFcF778ma9c8+k8OCwmFo397vyn4uas3cGPM+cPX/gwaw\nwD9aa98qvuYfgR7W2mE3+Dl25syZVx6npqaSmpp6s/qUUj7b9MMmvGyP13a9xpPtniQcCtOnZZ9A\nLW/89lvZD2bePPnQWRy+9OW4u6ysLLKysq48njVrVmxG5saYScB04HFr7aUbXKcjc6US2OmLp1m0\ndRFutktS9SSckMO45HHcVis4023XhC9JS5Pj7u6+27+aYnUDdBDwb8Cj1tqTN7lWm7lSAWCtZf2B\n9XgRj/UH1jOq8yjCoTBdmgUrfrl9u0zBLFsG/frJaD01NfY3TGPVzPcBNYGSRv6ltda5zrXazJUK\nmMO5h3lh0wu8sOkF2jZqSzgUZljHYdSsXtPv0qImNxdeeklG64WF0tQnTJCbp7GgoSGlVMzkF+az\nas8q3IhLzvEcpnafyozQDFo2TPD45VWsldW4ngfvvgsjRkhj79atcp9Xm7lSyhe7ftxFZiSTl7a/\nRJ+WfXBCDv3b9qeaScD45XUcPVp63N0990hTHzECateO/nNpM1dK+erc5XO8vONlMrIzyLuUR3oo\nncndJnNH3QSKX95EQQGsXi2j9U2bJF06Ywa0aRO959BmrpSKC9ZaNhzegJvtsmrPKoZ2GIqT4pBy\nd0qgljd+/bUEkRYuhJQUGa0/+WTFj7vTZq6Uijsnzp9g/ub5eBGPRnUa4YQcRncZTd2kun6XFjUX\nLsCrr8oN02PHZD+YKVPgzjt/2c/TZq6UiltFtoj3vn4PN+Ly+aHPmZA8gfRQOu2bxFn8soIiEZmC\neeMNOZTacaB371tb3qjNXCmVEA6ePsjsyGzmbZlHcrNkwqEwQ9oPoUa1Gn6XFjU//QQLFkhjr1NH\nmvrYsVC/HOdvazNXSiWUSwWXeH3X67jZLgdPHyStZxrTekzj7tt8jF9GWVGRHHOXkQEffSQNPRyW\nTUivR5u5UiphbTu2DS/bY1nOMvq36Y+T4tC3Vd9A3TA9dAheeEG+2reX0frQoVDzmryVNnOlVMLL\nvZTL4q2L8SIeRbaIcCjMhK4TaFg7RvHLGLh8GVaulCmY3bth2jTZE+bee+XPtZkrpQLDWsvH336M\nF/F475v3GNFxBE6KQ7fmlRy/jLGcHGnqS5dC374yWh8wQJu5UiqAjp49ytxNc5m9cTb3NrgXJ8Vh\neMfh1K5RCfFLn5w9Kw09IwO2bdNmrpQKsIKiAlbvXY0bcdn8w2Ymd5tMeiid+xrd53dpUWMtVKum\nzVwpVUXsO7mPzEgmC7cu5KF7HuK3Kb9lULtBgTjuTufMlVJVzvn887yy4xXciMuJ8yeuHHfXtF5T\nv0v7xbSZK6WqtMiRCF62xxu732Dw/YNxUhx6t+idcMsbtZkrpRRw6sIpFm5ZiBfxqJNUByfkMDZ5\nLPVrliN+GQe0mSul1FWKbBEf7P8AL+Lx0cGPGNNlDE6KQ8emN4hfxgFt5kopdR2HzhxizsY5zN08\nl/Z3tMdJcRjaYWhcHncX02ZujPkfwP8BmlhrT13nGm3mSqm4crnwMit3r8TNdtl7ci/TekwjrWca\nLRq08Lu0K2LWzI0xLYC5QHugpzZzpVQiyjmeQ2YkkyXbl9C3dV+ckEO/Nv18P+4uls18OfD/AavQ\nZq6USnBnL59lybYluBGXC/kXCIfCTOo2iUZ1GvlST0yauTFmCJBqrf2DMeYA2syVUgFhreWL77/A\nzXZZvW81z3V4DifFoefdPWNaR9SauTHmfaDZ1b8FWOB/Af8A9LfW5hU385C19uR1fo42c6VUQvrx\n3I+8uPlFMiOZNKvfjHAozMhOI6mTVKfSn7vSR+bGmM7AB8B5pMG3AA4DD1lrj5dxvZ05c+aVx6mp\nqaSmpv6i51ZKKT8UFhXyztfv4Ga7ZB/JZmLXiaSH0mnXuF3UniMrK4usrKwrj2fNmhXbpYnFI/Me\n1tqfrvPnOjJXSgXG/p/2Mzsym/lb5tP9ru44IYfBDwyO+nF3MV9nbozZj0yz6Jy5UqrKuFhwkeU5\ny/EiHt/nfi/7wfSYSvP6zaPy8zU0pJRSMbb5h814EY/lO5czsO1AnBSHR1o+UqH9YLSZK6WUT05f\nPM3irYtxIy7VTXXCoTDju46nQa0Gt/yztJkrpZTPrLVkHczCjbh8sP8DRnUaRTglTHKz5HL/DG3m\nSikVR47kHWHuprnM2TiH1re3xklxGPbgMGrVqHXD79NmrpRScSi/MJ9Ve1bhRTy2H9/O1O5TmdFz\nBq1ub1Xm9drMlVIqzu05sQcv4rF422IevvdhnBSHAW0H/Gw/GG3mSimVIM5dPseyHcvIyM7gzKUz\npPdMZ3L3yTSp20SbuVJKJRprLRsOb8CLeKzas4oh7Yew6NlF5Wrm0Y0qKaWU+sWMMfRq0YteLXpx\n8vxJ5m+ZX/7v1ZG5UkrFr/JOs/i767pSSqmo0GaulFIBoM1cKaUCQJu5UkoFgDZzpZQKAG3mSikV\nANrMlVIqALSZK6VUAGgzV0qpANBmrpRSAVDhZm6M+W/GmF3GmO3GmD9HoyillFK3pkLN3BiTCjwN\ndLHWdgH+bzSK8lNWVpbfJZSL1hk9iVAjaJ3Rlih1lldFR+Zh4M/W2gIAa+2Jipfkr0T5P1jrjJ5E\nqBG0zmhLlDrLq6LN/AHgUWPMl8aYD40xoWgUpZRS6tbcdD9zY8z7QLOrfwuwwP8q/v5G1tpexpgU\n4FWgTWUUqpRS6voqtJ+5MWYN8L+ttR8VP/4a+JW19mQZ1+pm5kop9QvE4qShlcDjwEfGmAeApLIa\neXmLUUop9ctUtJnPB+YZY7YDl4AJFS9JKaXUrYrZsXFKKaUqT0wToMaYfy0OGG0xxrxujGkQy+cv\nL2PMcGPMDmNMoTGmh9/1XM0YM8gYs9sYs9cY8z/9rqcsxpgXjTHHjDHb/K7lRowxLYwx640xOcWh\nt9/7XVNZjDG1jDEbjDGbi2v9F79ruh5jTDVjzCZjzCq/a7kRY8xBY8zW4r/Tr/yupyzGmIbGmOXF\nPTPHGPOrG10f6zj/WqCTtbYbsA/4Y4yfv7y2A88CH/ldyNWMMdWA/wIGAp2A0caYDv5WVab5SI3x\nrgD4g7W2E9Ab+G08/n1aay8Bj1lruwPJwOPGmId9Lut6/juw0+8iyqEISLXWdrfWPuR3MdfxF2CN\ntfZBoCuw60YXx7SZW2s/sNYWFT/8EmgRy+cvL2vtHmvtPmQZZjx5CNhnrf3WWpsPLAOe8bmmv2Kt\n/RT4ye86bsZae9Rau6X412eRfyz3+FtV2ay154t/WQv5dxt3f7/GmBbAU8Bcv2spB0Mc701VPGvx\niLV2PoC1tsBam3uj7/HzxUwB3vHx+RPRPcChqx5/T5w2n0RjjGkNdAM2+FtJ2YqnLzYDR4Esa208\njn7/Hfg7JIcS7yzwvjEm2xgz3e9iynAfcMIYM7942mqOMabOjb4h6s3cGPO+MWbbVV/bi//36auu\n+Ucg31q7NNrPH806VdVgjKkPvAb89+IRetyx1hYVT7O0QFLXff2u6WrGmMHAseJPOob4+1R7rYet\ntT2QTxK/Ncb08buga9QAegAZxXWeB/7+Zt8QVdba/jf6c2PMJOQv8PFoP/etuFmdceow0PKqxy2K\nf0/9QsaYGkgjX2ytfdPvem7GWptrjFkNhIivezoPA0OMMU8BdYDbjDGLrLVxuVzZWvtD8f/+aIxZ\ngUxhfupvVT/zPXDIWhspfvwacMMFD7FezTII+Rg2pPimTiKIpxFGNtDOGNPKGFMTGAXE66qBRBid\nAcwDdlpr/+J3IddjjGlijGlY/Os6QH9gi79V/Zy19h+stS2ttW2Q/y7Xx2sjN8bULf40hjGmHjAA\n2OFvVT9nrT0GHCoOYwL04yY3lmM9Z/7/gPrIXNUmY4wb4+cvF2PMUGPMIaAX8LYxJi7m9q21hcDv\nkFVBOcAya+0N73D7wRizFPgceMAY850xZrLfNZWleEXIWGR1yObi/yYH+V1XGe4CPiyeM/8SWGWt\nXedzTYmsGfDpVX+fb1lr1/pcU1l+DywxxmxBVrPccEmqhoaUUioA4nZpjlJKqfLTZq6UUgGgzVwp\npQJAm7lSSgWANnOllAoAbeZKKRUA2syVUioAtJkrpVQA/P96XjjdwgzWMQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1068d59d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def cost(ws):\n",
    "    b = ws[0]\n",
    "    w = ws[1:]\n",
    "    return np.linalg.norm(w)**2\n",
    "\n",
    "\n",
    "def constraints(ws):\n",
    "    b = ws[0]\n",
    "    w = ws[1:]\n",
    "    #print classes*(np.dot(data, w) + b) - 1\n",
    "    return classes*(np.dot(data, w) + b) - 1\n",
    "\n",
    "cons = ({'type': 'ineq',\n",
    "         'fun' : constraints})\n",
    "\n",
    "res = minimize(cost, [1., 2., 3.], method=\"SLSQP\", constraints=cons)\n",
    "\n",
    "print res \n",
    "b = res.x[0]\n",
    "w = res.x[1:]\n",
    "m = -w[0]/w[1]\n",
    "c = -b/w[1]\n",
    "\n",
    "xs = np.linspace(-2, 6, 100)\n",
    "ys = m*xs + c\n",
    "plt.plot(data[:, 0], data[:, 1], '.r')\n",
    "plt.plot(xs, ys)\n",
    "\n",
    "m = -w[0]/w[1]\n",
    "cl = -(b-1)/w[1]\n",
    "xs = np.linspace(-2, 6, 100)\n",
    "yls = m*xs + cl\n",
    "plt.plot(xs,yls)\n",
    "\n",
    "m = -w[0]/w[1]\n",
    "cr = -(b+1)/w[1]\n",
    "xs = np.linspace(-2, 6, 100)\n",
    "yrs = m*xs + cr\n",
    "plt.plot(xs,yrs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "15) What is a slack variable?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "16) How do we penalize the slack variable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "17) Plot soft margin cost function along with logistic cost."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "18) How can we view our new and penalized cost function as a regularization?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "19) Try your new cost function with these data points."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAEACAYAAABBDJb9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADe1JREFUeJzt3WGIZWd9x/Hfb51NCGTaFxVUskSRUsrMoImITAm7HlOC\nS4REwReRFqmFvmnFgFRskxeZvhHaN1Za+so0YCHkhS2YRNImNTnZ0U5S0SzuzGzivtJozYISulcC\nkon/vjh3ttvp7s65c557z73/8/3AZc6dPfc5z+HO/s5zn/M8z3VECACw2I71XQEAQHeEOQAkQJgD\nQAKEOQAkQJgDQAKEOQAkUCzMbR+z/X3bj5cqEwDQTsmW+f2SdguWBwBoqUiY2z4h6W5JXy1RHgBg\nMqVa5l+W9AVJTCcFgB50DnPbH5N0MSLOSvL4AQCYIXddm8X2lyT9oaQ9STdJWpb0LxHx6QP70WoH\ngCOIiEMbyZ1b5hHxQETcGhHvlXSfpGcPBvkV+6Z9PPTQQ73XodXj0iXF+9+vWFpqfl66lOv8Mr93\nnN8gz68txpkPzfa2tLMj7e1Ju7vNNoCFVzTMI+L5iLinZJkobG1NWl2Vjh+XVlaabQALb6nvCmRR\nVVXfVWhneVna3Gxa5KurzfMWFub8jiDzuUmc31B0vgHa+kB2zOpYAJCFbcUsboACAPpHmANAAoQ5\nACRAmANAAoQ5ACRAmANAAoQ5ACRAmANAAoQ5ACRAmANAAoQ5ACRAmANAAoQ5ACRAmANAAoQ5ACRA\nmANAAoQ5ACRAmANAAoQ5ACRAmANAAoQ5ACRAmANAAoQ5ACRAmANAAoQ5ACRAmANAAktdC7B9o6Qz\nkm4YP74REQ90LRcA0F7nMI+IX9n+SES8Yfttkr5j+46I+E6B+gEAWijSzRIRb4w3bxyX+XqJclHQ\naCRtbTU/AaRTJMxtH7P9kqTXJNURsVuiXBQyGkknT0qnTjU/CXQgnVIt819HxO2STkg6ZfvDJcpF\nIdvb0s6OtLcn7e422wBS6dxnfqWIuGT7m5I+KOn5g/++sbFxebuqKlVVVfLwuJa1NWl1tQnylZVm\nG7M3GjUX1rU1aXm579pgTtV1rbquJ36dI6LTgW2/XdKbEfHftm+S9G+S/ioivnVgv+h6LHQwGjUt\n8tVVgqQP+11d++/B5ibvA1qxrYjwYfuV6GZ5l6Tnxn3mL0h6/GCQYw4sL0vr6wRIX+jqwpR1bpm3\nPhAtcwzZfst8v6uLljlaatsyJ8yBWaGrC0dAmANAArPsMwcA9IwwB4AECHMASIAwB4AECHMASIAw\nB4AECHMASIAwB4AECHMASIAwB4AECHMASIAwB4AECHMASIAwB4AECHMASIAwB4AECHMASIAwB4AE\nCHMASIAwB4AECHMASIAwB4AECHMASIAwB4AECHMASIAwB4AECHMASKBzmNs+YftZ2zu2z9n+XImK\nAQDac0R0K8B+p6R3RsRZ2zdL+p6keyPi5QP7RddjAcDQ2FZE+LD9OrfMI+K1iDg73v6lpPOSbula\nLgCgvaJ95rbfI+k2SS+WLBcAcH1LpQoad7F8XdL94xb6/7OxsXF5u6oqVVVV6vAAkEJd16rreuLX\nde4zlyTbS5KelPRURHzlGvvQZw4AE2rbZ14qzL8m6ecR8fnr7EOYA8CEZhbmtu+QdEbSOUkxfjwQ\nEf96YD/CHAAmNNOWeRuEOQBMbmZDEwEA/SPMASABwhwAEiDMASABwhwAEiDMASABwhwAEiDMASAB\nwhwAEiDMASABwhwAEiDMASABwhwAEiDMASABwhwAEiDMASABwhwAEiDMASABwhwAEiDMASABwhzz\nYzSStraanwAmQphjPoxG0smT0qlTzc+jBDoXAwwYYY75sL0t7exIe3vS7m6zPYkSFwNggRHmmA9r\na9LqqnT8uLSy0mxPouvFAFhwjojZHMiOWR0LC2o0akJ4dVVaXp78tSdPNkG+siJtbk5eBjCHbCsi\nfOh+hDnS6HIxAOYUYQ4ACbQNc/rMASCBImFu+2HbF23/oER5AIDJlGqZPyLpo4XKAgBMqEiYR8S3\nJb1eoixg5phshAToM8fiKRm+TDZCEkuzPNjGxsbl7aqqVFXVLA+PDPbDd38IYtfx5FebbLS+Xq6+\nwITqulZd1xO/rtjQRNvvlvRERLzvGv/O0ER0t7XVtKL39prZomfOdAvfoU42Go2aC9na2jDOd4H1\nMTTR4wcwPV2n/R+0vNwE+JkzwwpyupbSKdIyt/2opErSb0m6KOmhiHjkwD60zFEGMz27Kf3pBlPF\nDFAAVzfUrqUFRZgDuDY+3SwMwhwAEmBtFgAYEMIcABIgzAEgAcIcABIgzAEgAcIcABIgzBcMq7UC\nuBrCfIGwpAaAayHMF8jVVmsFAIkwXyilFwwEkAfT+RcMS2oAw8LaLACQAGuzAMCAEOYAkABhDgAJ\nEOYAkABhDgAJEOYAkABhDgAJEOYAkABhDgAJEOYAkABhjiNjbXVgfhDmOBLWVgfmC2GOI2FtdWC+\nEOY4EtZWB+ZLkSVwbZ+W9LdqLg4PR8RfX2UflsBNhrXVgemb2Xrmto9J+qGk35f0X5K+K+m+iHj5\nwH6EOQBITUtoe7v5iHtIS2iW65l/SNKFiPhRRLwp6TFJ9xYoFwDymdLogRJhfoukV694/pPx7wAA\nB01p9MBSkVJa2tjYuLxdVZWqqprl4QGgf/ujB3Z3rzp6oK5r1XU9cbEl+szXJW1ExOnx87+QFAdv\ngtJnDgBjE4wemOUN0LdJekXNDdCfSfpPSZ+KiPMH9iPMAWBCM7sBGhFvSfqspKcl7Uh67GCQA1gc\nLNOwmIqMM291IFrmwNzbH2ix3wOwuckcgr7NcmgigCRYpmFxEeYALmOZhsVFNwuA/4NlGubLzEaz\ntEWYA8Dk6DMHgAEhzAEgAcIcABIgzAEgAcI8IWbwAcNDmCfDFy0Dw0SYJ8MMPmCYCPNkmMEHDBOT\nhhKa1gy+Cb62EEAhzABFUaymB/SDGaAoir54YL4R5miFvnhgvtHNgtZYTQ+YPfrMASAB+swBYEAI\ncwBIgDAHgAQI86RYbAsYFsI8IRbbGiCu3oNHmCfEBJ+B4eoNEeYpMcFnYLh6Q4wzT4sJPgOy3zLf\n3W2u3iyckwqThoAh4eqdFmEOAAnMZAao7U/a3rb9lu0PdCkLAHB0XW+AnpP0CUnPF6gLAOCIlrq8\nOCJekSTbh34EAABMD0MTASCBQ1vmtp+R9I4rfyUpJD0YEU9Mq2IAgPYODfOIuKvUwTY2Ni5vV1Wl\nqqpKFQ0AKdR1rbquJ35dkaGJtp+T9OcR8b3r7MPQRACY0KyGJn7c9quS1iU9afupLuUBAI6GSUMA\nMMf42jgAGBDCHAASIMwBIAHCHAASIMwBIAHCHAASIMwBIAHCHFhwo5G0tcX3OA8dYQ4ssP2v/zx1\nqvlJoA8XYQ4ssO3t5qs/9/aa73Pe2em7RugLYd4CH2Mxr9bWmu9wPn5cWllptjFMrM1yiP2Psftf\nfL65yZefY76MRv/798nfZj5t12YhzA+xtdX0R+7tNa2fM2ek9fW+awVgKFhoqxA+xgJYBLTMW+Bj\nLIC+0M0CAAnQzQIAA0KYA0AChPmcYUw7gKMgzOcIU7MBHBVhPkeYmg3gqAjzOcKY9qOjewpDx9DE\nOcOY9smx5AIyY5w5BoMlF5AZ48wxGHRPAbTMkQTdU8iKbhYASIBuFgAYkE5hbvtvbJ+3fdb2P9v+\njVIVAwC017Vl/rSk1Yi4TdIFSX/ZvUqLqa7rvqswVZnPL/O5SZzfUHQK84j494j49fjpC5JOdK/S\nYsr+B5X5/DKfm8T5DUXJPvM/lvRUwfIAAC0tHbaD7WckvePKX0kKSQ9GxBPjfR6U9GZEPDqVWgIA\nrqvz0ETbfyTpTyTdGRG/us5+jEsEgCNoMzTx0Jb59dg+LekLkk5dL8jbVgYAcDSdWua2L0i6QdIv\nxr96ISL+tETFAADtzWwGKABgemY6AzTzJCPbn7S9bfst2x/ouz6l2D5t+2XbP7T9xb7rU5Lth21f\ntP2DvusyDbZP2H7W9o7tc7Y/13edSrF9o+0Xbb80Pr8v9V2nabB9zPb3bT9+2L6zns6feZLROUmf\nkPR83xUpxfYxSX8v6aOSViV9yvbv9luroh5Rc25Z7Un6fESsSvo9SX+W5f0b36P7SETcLul9ku60\nfUfP1ZqG+yXtttlxpmGeeZJRRLwSERfUDN3M4kOSLkTEjyLiTUmPSbq35zoVExHflvR63/WYloh4\nLSLOjrd/Kem8pFv6rVU5EfHGePNGNVmW6r20fULS3ZK+2mb/PhfaYpLR/LtF0qtXPP+JEoXBkNh+\nj6TbJL3Yb03KGXdBvCTpNUl1RLRqwS6QL6sZLdjqxmanoYlXk3mSUZtzA+aN7ZslfV3S/eMWegrj\nT/m3j++9PW37wxGRopvT9sckXYyIs7YrtfjEXzzMI+Ku6/37eJLR3ZLuLH3saTvs3BL6qaRbr3h+\nYvw7LAjbS2qC/J8i4ht912caIuKS7W9K+qDy3LO6Q9I9tu+WdJOkZdtfi4hPX+sFsx7Nsj/J6J7D\nJhktuCz95t+V9Nu23237Bkn3STr0rvqCsfK8X1fzj5J2I+IrfVekJNtvt/2b4+2bJN0l6Wy/tSon\nIh6IiFsj4r1q/t89e70gl2bfZ/53km6W9Mx4uM0/zPj4U2P747ZflbQu6UnbC38/ICLekvRZNaOQ\ndiQ9FhHn+61VObYflfQfkn7H9o9tf6bvOpU0Ht3xB2pGerw0/j93uu96FfIuSc+N+8xfkPR4RHyr\n5zr1iklDAJAAXxsHAAkQ5gCQAGEOAAkQ5gCQAGEOAAkQ5gCQAGEOAAkQ5gCQwP8AIUPcdirU8E4A\nAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x106392fd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "np.random.seed(9999)\n",
    "xp = np.random.randn(10, 2)\n",
    "xm = np.random.randn(10, 2) + [1, 1]\n",
    "plt.plot(xp[:, 0], xp[:,1], '.b')\n",
    "plt.plot(xm[:, 0], xm[:,1], '.r')\n",
    "data = np.concatenate([xp, xm], axis=0)\n",
    "classes = np.array([1]*10+[-1]*10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "20) What if we want a curved line? Draw contour plot."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-1 -1 -1  1  1 -1  1  1  1 -1 -1  1  1 -1  1  1  1 -1 -1 -1]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x105c14a90>]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAEACAYAAABVtcpZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFE5JREFUeJzt3X+QXeV93/H3RxaMa3vLQFwEAUPHdgxB2zj2TLEyNspt\nKbHsOBHtZBzw1DHu1GE6pvU0nRaR0mE7k7bWH526qZOmYMaDO8lQx06K4pogCNyKTOWAE4MtaSXk\nuCEgg9yU4G7czBSp3/5xr+Tteu/+0Dm7d++e92tmZ8+596vzHN25ez/3/HieJ1WFJKmbtox7ByRJ\n42MISFKHGQKS1GGGgCR1mCEgSR1mCEhSh7USAkl2JTma5Jkkty/y/PcleTDJU0m+luSWNtqVJDWT\npv0EkmwBngGuB74JPAncVFVH59XcBby6qu5I8nrgGLCtqk41alyS1EgbRwLXAser6tmqegW4H9i9\noOZFYGq4PAX8TwNAksZvawvbuAx4bt768wyCYb57gN9J8k3gdcBPt9CuJKmh9bowfAfwdFV9P/A2\n4JeSvG6d2pYkjdDGkcAJ4Ip565cPH5vvncC/AKiqP0zy34GrgS8v3FgSBzOSpFWqqpzLv2vjSOBJ\n4M1JrkxyPnATsG9BzSzwNwCSbAPeAnxj1Aaryp8Wfu66666x78Nm+vH19PXcqD9NND4SqKrTSW4D\n9jMIlXurajbJrYOn627gXwGfTvI0EOCfVNVLTduWJDXTxukgquq3gasWPPYf5i3/CfATbbQlSWqP\nPYY3sV6vN+5d2FR8Pdvl67kxNO4s1rYktdH2SZI2siTUGC8MS5Ntbg4OHhz8ljrGEFC3zc3BddfB\nzp2D3waBOsYQULcdOgSHD8OpU3DkyGBZ6hBDQN02PQ3bt8N558E11wyWpQ7xwrA0Nzc4Ati+Haam\nlq+XNpgmF4YNAUmacN4dJEk6J4aAJHWYISBJHWYISFKHGQKS1GGGgCR1mCEgSR1mCEhSh7USAkl2\nJTma5Jkkt4+o6SX5SpJDSR5ro11JUjONewwn2QI8A1wPfJPBnMM3VdXReTUXAP8N+LGqOpHk9cPZ\nxhbbnj2GJWkVxt1j+FrgeFU9W1WvAPcDuxfUfAD4fFWdgLPTTUqSxqyNELgMeG7e+vPDx+Z7C3BR\nkseSPJnkgy20K0lqqJWJ5lfYztuBvw68FjiY5GBVfX2x4pmZmbPLvV7PuUglaZ5+v0+/329lW21c\nE9gBzFTVruH6HqCqau+8mtuBV1fVPx+ufwp4sKo+v8j2vCYgSasw7msCTwJvTnJlkvOBm4B9C2oe\nAN6V5FVJXgO8A5htoW1JUgONTwdV1ekktwH7GYTKvVU1m+TWwdN1d1UdTfIQ8FXgNHB3VR1p2rYk\nbShzc4MpS6enJ2aCIieVkaQ2zM3Bddd9d5a6xx9ftyAY9+kgSdKhQ4MAOHUKjhwZLE8AQ0CS2jA9\nDVdfDVu3wlVXDY4GJoAhIEltmrDT2YaAJLXh0CE4ehROn4ZjxzwdJEmdMj09OAV03nlwzTUTczrI\nu4MkqS1zc9+9O2gdbxFtcneQISBJE85bRCVJ58QQkKQOMwQkqcMMAUnqMENAkjrMEJCkDjMEJKnD\nDAFJ6jBDQJI6rJUQSLIrydEkzwznEx5V91eTvJLkb7XRriSpmcYhkGQL8Eng3cB24OYkV4+o+zjw\nUNM2JUntaONI4FrgeFU9W1WvAPcDuxep+/vA54BvtdCmJKkFbYTAZcBz89afHz52VpLvB26sqn8P\nnNMgR5Kk9m1dp3Y+Acy/VrBkEMzMzJxd7vV69Hq9NdkpSZpE/X6ffr/fyrYaDyWdZAcwU1W7hut7\ngKqqvfNqvnFmEXg98B3gZ6tq3yLbcyhpSVqFsc4nkORVwDHgeuAF4Ang5qqaHVH/aeC3quo3Rjxv\nCEjSKjQJgcang6rqdJLbgP0MrjHcW1WzSW4dPF13L/wnTduUJLXDmcUkacI5s5gk6ZwYApLUYYaA\nJHWYISBJHWYISFKHGQKS1GGGgCR1mCEgjTA3BwcPDn5Lm5UhIC1ibg6uuw527hz8Ngi0WRkC0iIO\nHYLDh+HUKThyZLAsbUaGgLSI6WnYvh3OOw+uuWawLG1Gjh0kjTA3NzgC2L4dpqbGvTfSaGMdSrpt\nhoAkrY4DyEmSzokhIEkdZghIUoe1EgJJdiU5muSZJLcv8vwHkjw9/PndJH+ljXYlSc00DoEkW4BP\nAu8GtgM3J7l6Qdk3gJ1V9VbgF4B7mrYrgb16pabaOBK4FjheVc9W1SvA/cDu+QVV9aWq+vZw9UvA\nZS20q46zV6/UXBshcBnw3Lz151n6Q/7vAg+20K46zl69UnNb17OxJH8N+DDwrqXqZmZmzi73ej16\nvd6a7pcm05levUeO2KtX3dLv9+n3+61sq3FnsSQ7gJmq2jVc3wNUVe1dUPdDwOeBXVX1h0tsz85i\nWjF79Upj7jGc5FXAMeB64AXgCeDmqpqdV3MF8DvAB6vqS8tszxCYQHNzg9Mz09N+GEvrbaw9hqvq\nNHAbsB84DNxfVbNJbk3ys8OyfwZcBPxykq8keaJpu1p/o+7E8QKtNLkcO0grcuaD/sypl8cf/+43\n/oMHBwFw6tRg1M0DB2DHjvHur9Qljh2kNbfUnTgOuyxNLo8EtCJnjgTO3Ikz/0jgzPNeoJXGw6Gk\ntS78oJc2JkNAkjrMawKSpHNiCEjzOCCdumYyQ8C/VK0B+zuoiyYvBPxL1RpxQLrv8ntWd0xeCPiX\nqjVif4cBv2d1y+SFgH+pWiNTU4P+DwcOfG8/iC7xe1a3TOYtot6wLq2Z5ToGauOxn4CkVvk9a7IY\nApLUYXYWkySdE0NAkjqslRBIsivJ0STPJLl9RM0vJjme5KkkP9xGu5KkZhqHQJItwCeBdwPbgZuT\nXL2g5j3Am6rqB4BbgV9p2q4kqbk2jgSuBY5X1bNV9QpwP7B7Qc1u4DMAVfV7wAVJtrXQtiSpgTZC\n4DLguXnrzw8fW6rmxCI1kqR15oVhSeqwrS1s4wRwxbz1y4ePLax5wzI1Z83MzJxd7vV69Hq9pvso\nSZtGv9+n3++3sq3GncWSvAo4BlwPvAA8AdxcVbPzat4LfLSqfjzJDuATVbVjxPbsLCZJq9Cks1jj\nI4GqOp3kNmA/g9NL91bVbJJbB0/X3VX1xSTvTfJ14DvAh5u2K210c3ODwdimpx16QRuXw0ZIa+DM\nIGxnxt9xEDatJYeNkDYYh2PWpDAEpDXgtBeaFJ4OktaIwzFrvTiUtCR1mNcEJEnnxBCQpA4zBCSp\nwwwBSeowQ0CSOswQkKQOMwSkCTI3BwcPDn5LbTAEpAlxZjyinTsHvw0CtcEQkCaE4xFpLRgC0oRw\nPCKtBYeNkCaI4xFpMY4dpFY4CYo0mcY2dlCSC5PsT3IsyUNJLlik5vIkjyY5nORrSf5Bkza1Nrzo\nKHVT02sCe4BHquoq4FHgjkVqTgE/V1XbgR8BPprk6obtqmVedJS6qWkI7AbuGy7fB9y4sKCqXqyq\np4bLfwbMApc1bFct86Kj1E2NrgkkeamqLhq1vkj9Xwb6wPQwEBar8ZrAmHjRUZpMTa4JbF3Bxh8G\nts1/CCjgzkXKR356J3kd8DngY6MCQOM1NQU7dox7LyStp2VDoKpuGPVckpNJtlXVySSXAN8aUbeV\nQQD8x6p6YLk2Z2Zmzi73ej16vd5y/0SSOqPf79Pv91vZVtPTQXuBl6pqb5LbgQuras8idZ8B/qSq\nfm4F2/R0kCStwtj6CSS5CPgs8AbgWeD9VfVykkuBe6rqfUneCRwAvsbgdFEBP19Vvz1im4aAJK2C\nncUkqcOcaF6SdE4MAUnqMENArXCyE2kyGQJqzHGHpMllCKgxxx2SJpchoMYcd0iaXN4iqlY47pA0\nPvYTkKQOs5+AJOmcGAKS1GGGgCR1mCEgSR1mCCzHrrCSNjFDYCl2hZW0yRkCS7ErrKRNzhBYil1h\nJW1yTWcWuxD4T8CVwB8xmFns2yNqtwBfBp6vqp9cYpsbq7OYXWElbXDj7Cy2B3ikqq4CHgXuWKL2\nY8CRhu2tv6kp2LHDAJC0KTUNgd3AfcPl+4AbFytKcjnwXuBTDduTJLWoaQhcXFUnAarqReDiEXX/\nBvjHDCaZlyRtEFuXK0jyMLBt/kMMPszvXKT8ez7kk/w4cLKqnkrSG/77Jc3MzJxd7vV69Hq95f6J\nJHVGv9+n3++3sq2mF4ZngV5VnUxyCfBYVf3ggpp/Cfxt4BTwF4Ap4Deq6mdGbHNjXRiWpA1unBeG\n9wG3DJc/BDywsKCqfr6qrqiqNwI3AY+OCgBJ0vpqGgJ7gRuSHAOuBz4OkOTSJF9ounOSpLXlpDKS\nNOGcVEaSdE4MAUnqMENAkjrMEJCkDjMEJKnDDAFJ6jBDQJI6zBCQpA4zBCSpwwwBSeowQ0CSOswQ\nkKQOMwQkaZzm5uDgwcHvMTAEJGlc5ubguutg587B7zEEgSEgafMZ87frFTt0CA4fhlOn4MiRwfI6\nMwQkbS4b4Nv1ik1Pw/btcN55cM01g+V11igEklyYZH+SY0keSnLBiLoLkvx6ktkkh5O8o0m7kjTS\nBvh2vWJTU/D443DgwOD31NS670LTI4E9wCNVdRXwKHDHiLp/C3xxOAn9W4HZhu1K0uI2wLfrVZma\ngh07xhIA0HB6ySRHgR+tqpNJLgH6VXX1gpq/CHylqt60wm06vaSkZubmBkcA27eP7cN1PTWZXrJp\nCLxUVReNWh8+9lbgbuAIg6OALwMfq6o/H7FNQ0CSVqFJCGxdwcYfBrbNfwgo4M5Fyhf79N4KvB34\naFV9OcknGJxGumtUmzMzM2eXe70evV5vud2UpM7o9/v0+/1WttX0SGAW6M07HfTY8Lz//JptwMGq\neuNw/V3A7VX1EyO2uTmOBObmBheopqc7cTgqaXyaHAk0vTC8D7hluPwh4IGFBVV1EnguyVuGD13P\n4NTQ5jVJt6hJ6rSmRwIXAZ8F3gA8C7y/ql5OcilwT1W9b1j3VuBTwHnAN4APV9W3R2xz8o8EDh4c\nBMCpU4M7FA4cGFz9l6Q1MLYLw2thU4TAmSOBI0cGt6iN6f5fSd1gCGxEHbtFTdL4GAKS1GHjvDAs\nSZNrUgaaW0OGgKRu8i4+wBCQ1FWTNNDcGjIEJHXTpA00t0a8MCypuzbJXXzeHSRJHebdQZKkc2II\nSFKHGQKS1GGGgCR1mCEgSR1mCEhShxkCktRhhoAkdVijEEhyYZL9SY4leSjJBSPq7khyOMlXk/xq\nkvObtCtJakfTI4E9wCNVdRXwKHDHwoIkVwIfAd5WVT8EbAVuatiuJKkFTUNgN3DfcPk+4MZFav4X\n8H+A1ybZCrwG+GbDdiVJLWgaAhdX1UmAqnoRuHhhQVX9KfCvgT8GTgAvV9UjDduVJLVg63IFSR4G\nts1/CCjgzkXKv2fktyRvBP4hcCXwbeBzST5QVb82qs2ZmZmzy71ej16vt9xuShqXubnB2PzT0xM9\nEuck6ff79Pv9VrbVaBTRJLNAr6pOJrkEeKyqfnBBzfuBG6rqI8P1DwLvqKrbRmzTUUSlSXFmdq4z\nwzE//rhBMAbjHEV0H3DLcPlDwAOL1BwDdiR5dZIA1wOzDduVtBE4O9fEaxoCe4Ebkhxj8OH+cYAk\nlyb5AkBVPQ18Bvh94GkGp5PubtiupI3A2bkmnpPKSGpmk8zONcmcWUySOsyZxSRJ58QQkKQOMwQk\nqcMMAUnqMENAkjrMEJCkDjMEJKnDDAFJ6jBDQJI6zBCQpA4zBCSpwwwBSeowQ0CSOswQkKQOaxQC\nSX4qyaEkp5O8fYm6XUmOJnkmye1N2pQktafpkcDXgL8J/NdRBUm2AJ8E3g1sB25OcnXDdrUCbU1E\nrQFfz3b5em4MjUKgqo5V1XEGU0aOci1wvKqerapXgPuB3U3a1cr4R9YuX892+XpuDOtxTeAy4Ll5\n688PH5MkjdnW5QqSPAxsm/8QUMA/rarfWqsdkyStvVbmGE7yGPCPquoPFnluBzBTVbuG63uAqqq9\nI7blBMOStErnOsfwskcCqzBqB54E3pzkSuAF4Cbg5lEbOdf/iCRp9ZreInpjkueAHcAXkjw4fPzS\nJF8AqKrTwG3AfuAwcH9VzTbbbUlSG1o5HSRJmkxj7TFsZ7N2Jbkwyf4kx5I8lOSCEXV/lOTpJF9J\n8sR67+dGt5L3W5JfTHI8yVNJfni993FSLPdaJvnRJC8n+YPhz53j2M9JkeTeJCeTfHWJmlW9N8c9\nbISdzdq1B3ikqq4CHgXuGFH3f4FeVb2tqq5dt72bACt5vyV5D/CmqvoB4FbgV9Z9RyfAKv52D1TV\n24c/v7CuOzl5Ps3g9VzUubw3xxoCdjZr3W7gvuHyfcCNI+rC+L8AbFQreb/tBj4DUFW/B1yQZBta\naKV/u94MskJV9bvAny5Rsur35iR8ENjZbOUurqqTAFX1InDxiLoCHk7yZJKPrNveTYaVvN8W1pxY\npEYr/9v9keGpi/+S5Jr12bVNa9XvzTZvEV2Unc3atcTrudi51FFX/d9ZVS8k+UsMwmB2+A1DWm+/\nD1xRVf97eCrjPwNvGfM+dcqah0BV3dBwEyeAK+atXz58rJOWej2HF4y2VdXJJJcA3xqxjReGv/9H\nkt9kcNhuCAys5P12AnjDMjVawWtZVX82b/nBJL+c5KKqemmd9nGzWfV7cyOdDlq2s1mS8xl0Ntu3\nfrs1UfYBtwyXPwQ8sLAgyWuSvG64/Frgx4BD67WDE2Al77d9wM/A2R7xL585Daf/z7Kv5fzz1Umu\nZXDbugGwtDD683LV7801PxJYSpIbgX8HvJ5BZ7Onquo9SS4F7qmq91XV6SRnOpttAe61s9lIe4HP\nJvk7wLPA+2HQeY/h68ngVNJvDofn2Ar8alXtH9cObzSj3m9Jbh08XXdX1ReTvDfJ14HvAB8e5z5v\nVCt5LYGfSvL3gFeAPwd+enx7vPEl+TWgB3xfkj8G7gLOp8F7085iktRhG+l0kCRpnRkCktRhhoAk\ndZghIEkdZghIUocZApLUYYaAJHWYISBJHfb/AHM/iPNWt/8AAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x105c14450>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "np.random.seed(9999)\n",
    "data = np.random.random((20,2))*2-1\n",
    "def is_pos(x):\n",
    "    return 1 if np.linalg.norm(x-[0.1,0.1])<0.6 else -1\n",
    "classes = np.array([is_pos(x) for x in data])\n",
    "print classes\n",
    "plt.plot(data[classes==1,0], data[classes==1,1], '.b')\n",
    "plt.plot(data[classes==-1,0], data[classes==-1,1], '.r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "21) What are the support vectors?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}