sklearn_pr_ap.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Precision-recall curve and Average Precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scikit-learn version: 0.20.0\n",
      "matplotlib version: 2.1.2\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import sklearn\n",
    "import matplotlib\n",
    "from sklearn import metrics\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.utils.fixes import signature\n",
    "\n",
    "print('scikit-learn version: {}'.format(sklearn.__version__))\n",
    "print('matplotlib version: {}'.format(matplotlib.__version__))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 20 20\n"
     ]
    }
   ],
   "source": [
    "model_1_score = [0.4325, 0.3498, 0.2368, 0.2601, 0.1698, 0.211 , 0.1913, 0.8441,\n",
    "                 0.098 , 0.1682, 0.1844, 0.3937, 0.1746, 0.295 , 0.3164, 0.1856,\n",
    "                 0.1353, 0.2147, 0.1898, 0.2257]\n",
    "model_2_score = [0.502, 0.502, 0.502, 0.502, 0.502, 0.502, 0.502, 0.502,\n",
    "                 0.502, 0.502, 0.502, 0.502, 0.502, 0.502, 0.502, 0.502,\n",
    "                 0.502, 0.502, 0.502, 0.502]\n",
    "y_true = [1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
    "\n",
    "print(len(model_1_score), len(model_2_score), len(y_true))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model_1 AP = 0.30556\n",
      "Model_2 AP = 0.10000\n"
     ]
    }
   ],
   "source": [
    "model_1_ap = metrics.average_precision_score(y_true, model_1_score)\n",
    "model_2_ap = metrics.average_precision_score(y_true, model_2_score)\n",
    "print(\"Model_1 AP = {:.5f}\".format(model_1_ap))\n",
    "print(\"Model_2 AP = {:.5f}\".format(model_2_ap))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.  0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.\n",
      " 0. ]\n",
      "[0.11111111 0.05882353 0.0625     0.06666667 0.07142857 0.07692308\n",
      " 0.08333333 0.09090909 0.1        0.11111111 0.125      0.14285714\n",
      " 0.16666667 0.2        0.25       0.33333333 0.5        0.\n",
      " 1.        ]\n"
     ]
    }
   ],
   "source": [
    "precision_1, recall_1, thresholds_1 = metrics.precision_recall_curve(y_true, model_1_score)\n",
    "print(recall_1)\n",
    "print(precision_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 0.]\n",
      "[0.1 1. ]\n"
     ]
    }
   ],
   "source": [
    "precision_2, recall_2, thresholds_2 = metrics.precision_recall_curve(y_true, model_2_score)\n",
    "print(recall_2)\n",
    "print(precision_2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use `line plot` to plot PR-curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl0nOV96PHvb/ZNo12yLFmWbXmRN2wjMGYxi21CCIUkkIADYW24bW/anpum96bn3tOkuT3p7e1pTpte0hYChEBYkkBTJ4EQbOzYGAw2GIx3y6tGi7VY6yxan/vHOxaysa2xpdHMSL/POTrWO/POzPPK0vt7n+f5vb9HjDEopZRSALZUN0AppVT60KCglFJqiAYFpZRSQzQoKKWUGqJBQSml1BANCkoppYZoUFBKKTVEg4JSSqkhGhSUUkoNcaS6AReroKDAVFRUpLoZSimVUd5///0WY0zhSPtlXFCoqKhgx44dqW6GUkplFBE5nsh+OnyklFJqiAYFpZRSQzQoKKWUGpJxcwpKqcmlr6+PUChELBZLdVMygsfjoaysDKfTeUmv16CglEproVCIrKwsKioqEJFUNyetGWNobW0lFAoxY8aMS3qPpA0fichTItIkIrvP87yIyA9EpEZEdonIsmS1RSmVuWKxGPn5+RoQEiAi5Ofnj6pXlcw5hR8Dt1zg+c8Cs+NfjwL/msS2KKUymAaExI32Z5W0oGCM2QycusAudwA/MZZtQI6IlCSrPS3NHWzZ20hf/0CyPkIppTJeKrOPSoHaYduh+GOfIiKPisgOEdnR3Nx8SR9WcyrG1h4HT+xvZ19bD7o2tVJKfVpGpKQaYx43xlQbY6oLC0e8S/ucrvIZ7g034LHb+M9jXTxf00FTtH+MW6qUmohEhPvuu29ou7+/n8LCQm677baLep+KigpaWlpGtc/DDz9MUVERCxcuvKjPTlQqg0IdMG3Ydln8saSZNtDDg7Oz+Mw0Py3RAZ7e387varuJ9g8m82OVUhnO7/eze/duotEoAG+88Qalpecc2Ei6Bx98kN/+9rdJe/9UpqSuA74uIi8Cy4EOY0xDsj/UJsLSAi/zcty81Rjhg+YYe9t6WFniY0mBB5tOaCmVvmpOQHdkbN8z4IPK8hF3u/XWW/nNb37DXXfdxQsvvMDatWvZsmULAKdOneLhhx/myJEj+Hw+Hn/8cRYvXkxraytr166lrq6OFStWnDFs/dxzz/GDH/yA3t5eli9fzg9/+EPsdvuI7Vi5ciXHjh275MMdSTJTUl8A3gHmikhIRB4RkT8SkT+K7/IqcASoAZ4A/iRZbTkXr8PGmrIAD83Locjr4HehME/vb+dEV994NkMplSHuueceXnzxRWKxGLt27WL58uVDz337299m6dKl7Nq1i+9973vcf//9APzN3/wN1157LXv27OELX/gCJ06cAGDfvn289NJLbN26lQ8//BC73c5Pf/rTlBzX2ZLWUzDGrB3heQP812R9fqKKvA7WVgY50NHLm6Ewz9d0MC/HxY2lfrJdI0dtpdQ4SuCKPlkWL17MsWPHeOGFF7j11lvPeO6tt97i5ZdfBuCmm26itbWVzs5ONm/ezCuvvALA5z73OXJzcwHYsGED77//PldccQUA0WiUoqKicTya89M7mrEmkebluJkVdPHuySjbTkao6ejlqmIfy4u9OG06pKSUgttvv51vfvObbNq0idbW1kt+H2MMDzzwAH/3d383hq0bGxmRfTRenDbh2hIfX5ufS2W2i7caIzyxr40D7ZrCqpSyMn++/e1vs2jRojMev+6664aGfzZt2kRBQQHBYJCVK1fy/PPPA/Daa6/R1tYGwKpVq/jFL35BU1MTYM1JHD+e0HIHSadB4RyyXXY+PyPI2sogbpvwH0e7eLGmk2ZNYVVqUisrK+PP/uzPPvX4d77zHd5//30WL17Mt771LZ555hnAmmvYvHkzCxYs4JVXXqG83Br+mj9/Pn/7t3/LzTffzOLFi1mzZg0NDYnl2axdu5YVK1Zw4MABysrKePLJJ8fuAAHJtCvg6upqc0krr9U2wpEQXLsUEpjhP23QGHa2xNjSEKFnwLCs0MN1U3x4HBpPlRoP+/bto6qqKtXNyCjn+pmJyPvGmOqRXqtzCiOwiXB5oZf5uW42N3ySwnp9iZ/F+W5NYVVKTSgaFBLkddj4zLQAS/I9rK/r5re13exsibKmLEBZ4NLqliul1Lm0trayatWqTz2+YcMG8vPzk/rZGhQuUrHPwVcqs9nf3subdWGeO9TB/Fw3N071kaUprEqpMZCfn8+HH36Yks/WoHAJRISqXCuFdVtThHdPRjnU0cOKYh9XFnlxaAqrUipDaVAYBZddWFniZ3GehzfrwmxuiLCrNcaqMj+VQZfWgFdKZRxNoRkDOW47X5wZ5J5ZQRw24eUjXfzscCctMU1hVUplFg0KY6gi6OKheTmsLvVTH+nnqX3tbAh1ExvQKqxKqcygQWGM2UWoLvLyX6pyWZzvYXtzjMf3tvFRa0zvilYqQ6XLegq1tbXceOONzJ8/nwULFvDP//zPF/X5idCgkCQ+p41bygM8ODeHXLed105088zBDurCWoVVqUyTLuspOBwO/vEf/5G9e/eybds2HnvsMfbu3Tu2nzGm76Y+ZYrPwX2zs9nb1sPG+gjPHuxgYZ6bG6b6CTg1Jit1MdaHujk5xuVmir0OVpcFRtwvHdZTKCkpoaTEWso+KyuLqqoq6urqmD9//ih+AmfSs9I4EBEW5Hl4tCqXFcVe9rX18PjeNradjNA/qENKSmWCdFtP4dixY+zcufOMdowF7SmMI5dduH6qn8X5HjbUhdlUH+Gj1hirSgNUZrtS3Tyl0l4iV/TJkk7rKXR3d3PnnXfyT//0TwSDwbE4vCEaFFIg123nrplBjnT2sj4U5hdHOpkVdLKqNECeR++KVipdpcN6Cn19fdx5553ce++9fPGLX7zkNpyPDh+l0Mygi0fm5XBTqZ9Qdz8/2t/GxrowPZrCqlRaSvV6CsYYHnnkEaqqqvjGN74xloc2RHsKKWa3CVcWeVmQ6+b39WHebYqy+1SMG6b6WZjn1ruilUojF1pP4eGHH2bx4sX4fL4z1lNYu3YtCxYs4Oqrrz7negqDg4M4nU4ee+wxpk+ffsHP37p1K88++yyLFi1iyZIlAHzve9/71HDWaOh6CmmmIdzHG6Ew9ZF+pvocrCnzU+LXKqxq8tL1FC7eaNZT0OGjNFPid/LVOdl8rjxAR+8Azxzs4DfHuwj36ZCSUir5dPgoDYkIi/I9zMlx8XZjlO3NUQ6293L1FC/VhV7sWoVVqQlN11NQ5+S227ix1M9l+R421HWzsT7CR609rC7zMzOoKaxq8jDGTKr5tdGspzDaKQEdPsoAeR47X5qVzV0zgxgMPzvcyS8Od9LWM5DqpimVdB6Ph9bWVq0dlgBjDK2trXg8nkt+D+0pZJDKbBcVWbnsaI7ydmOUH+1r44oiL1cX+3DZJ89VlJpcysrKCIVCNDc3p7opGcHj8VBWVnbJr9egkGEcNuGqYh8L8zxsqg+z7WSU3ad6uHGqj/m5msKqJh6n08mMGTNS3YxJQ4ePMlTAaeO26Vl8dU42AaeNXx3v5rlDHTRGdGEfpdSl06CQ4Ur9Th6Yk81nywO09Qzw4wPtvHaii4imsCqlLoEOH00AIsJl+R7m5rjY2hDh/eYY+9t7uXaKj2WFHuw6pKSUSpD2FCYQj93GqrIAD1flMNXnYENdmKf3t3OsszfVTVNKZYikBgURuUVEDohIjYh86xzPl4vIRhHZKSK7RGTsCnhMYgUeB1+eFeTOmVn0DxpePNzJK0c6adcUVqXUCJIWFETEDjwGfBaYD6wVkbOXB/pfwM+MMUuBe4AfJqs9k42IMDvbzR9W5XJ9iY+jXb08sa+NzfVhegc031spdW7JnFO4EqgxxhwBEJEXgTuA4QuKGuD0ChHZQH0S2zMpOWzCiik+FuS52VQf4e2TUT4+1cONpX6qclyawqqUOkMyh49Kgdph26H4Y8N9B7hPRELAq8CfJrE9k1rQZef2iizunZ2NzyGsO9bFTw91cFJTWJVSw6R6onkt8GNjTBlwK/CsiHyqTSLyqIjsEJEdelfj6EwLOHlgbg63TAvQGk9hfb22m0i/prAqpZIbFOqAacO2y+KPDfcI8DMAY8w7gAcoOPuNjDGPG2OqjTHVhYWFSWru5GETYUmBh/9SlcuyQg8ftsR4fG8b7zdHGdT6MkpNaskMCtuB2SIyQ0RcWBPJ687a5wSwCkBEqrCCgnYFxonHYWNNWYCH5+VQ7HXwRshKYT3epSmsSk1WSQsKxph+4OvA68A+rCyjPSLyXRG5Pb7bXwBfE5GPgBeAB42WQhx3hV4H91QG+cKMLHoHDS/UdPIfRzvp6NUUVqUmm6Te0WyMeRVrAnn4Y3897Pu9wDXJbINKjIgwN8fNzKCL95qivNMY4XBHL8uLvVxV7MOpC/soNSlomQt1BqdNuGaKj4V5bjbVhdnaGOXj1h5uKvUzV1NYlZrwUp19pNJUtsvOHTOCfKUyG7dd+OWxLl6o6aQpqimsSk1kGhTUBZVnOXloXg43l/lpivbz9P52flfbTVRTWJWakHT4SI3IJsKyQi9VuW62NETY2RJjX1sPK6f6uCzfg02HlJSaMLSnoBLmddi4eVqAh+blUOh18HptmB8faOdEd1+qm6aUGiMaFNRFK/I6WFsZ5PMVWcT6Dc8f6uA/j3bSqSmsSmU8HT5Sl0REmJfrZla2i20nI7x7MkpNZy8rin1cWeTFoSmsSmUkDQpqVJw24boSP4vyPGysD7O5IcJHrTFWlfqZna0prEplGh0+UmMix23nCzOC3FMZxGkTXjnaxUuHO2nRFFalMooGBTWmKrJcPDwvhzVlfhoi/Ty5v531oW5imsKqVEbQ4SM15mwiXB5PYd1cH2FHc4y9bT1cX+JnUb5bU1iVSmPaU1BJ43PYuKU8wINzc8hz23mttpufHOggpCmsSqUtDQoq6ab4HNw7O5vbp2cR7h/kuUMd/OpYF119msKqVLrR4SM1LkSE+XluKk+nsDZFOdjRw9XFPq7QFFal0oYGBTWuXHZh5VQ/i/I9vFkX5venU1jL/FQGNYVVqVTT4SOVErluO3fODHL3rCB2EV4+0sXPD3fSGtMUVqVSSYOCSqkZQRcPV+WwqtRPXbifJ/e182ZdmJ4BTWFVKhV0+EilnF2EK4q8zM918/uGMO81RdlzKsb1U/0synPrkJJS40h7Cipt+J02bi3P4oG52eS47bx6opufHOygPqwprEqNFw0KKu2U+JzcNzub26YH6Ood5CcHO/j18S66+3RISalk0+EjlZZEhIV5HmZnu3inMcp7zVEOtvdyzRQv1YVe7JrCqlRSaE9BpTW33cYNpX7+cF4u0wIONtZHeHJ/O4c7elPdNKUmJA0KKiPkeex8aVY2X5oZBODnRzr5+eEOTsX0rmilxpIOH6mMMivbRUWWkx3NUbY2RvnR/jauLPSyYooXt12vcZQaLQ0KKuPYbcLyYh8L8jz8vj7MtqYou0/1cEOpjwW5msKq1GjopZXKWAGnjc9Nz+L+OdlkuWz8+ng3zx3qoCGiKaxKXSoNCirjTfU7uX9ONreWB2jvGeCZAx28eqKLsKawKnXRdPhITQgiwuJ8D3NzXGxtjLKjKcqBtl6uKfFxeaEHuw4pKZUQ7SmoCcVtt3FTqZ9HqnIo9Tt4sy7MU/vbOdqpKaxKJUKDgpqQ8j0OvjQryF0zgwwMGl463MnLRzpp79EUVqUuRIeP1IQlIlRmu6jIymV7U5S3T0Z4Yl8vVxZ5WVHsw2XXISWlzpbUnoKI3CIiB0SkRkS+dZ59viwie0Vkj4g8n8z2qMnJYRNWTPHx6Pxc5uW4eedklCf2tbHnVAxjTKqbp1RaSVpPQUTswGPAGiAEbBeRdcaYvcP2mQ38FXCNMaZNRIqS1R6lspx2/qAii6UFHtaHwvzqeDc7W2KsLgswxaedZqUguT2FK4EaY8wRY0wv8CJwx1n7fA14zBjTBmCMaUpie5QCoCzg5P652Xx2WoBTPQP8+EA7vz3RTURTWJVKrKcgIm7gTqBi+GuMMd+9wMtKgdph2yFg+Vn7zIm//1bADnzHGPPbc3z+o8CjAOXl5Yk0WakLsolwWYGVwvpWY4T3m2Psa+/huhIfywo82DSFVU1SifYU/hPrKr8fCA/7Gi0HMBu4AVgLPCEiOWfvZIx53BhTbYypLiwsHIOPVcricdhYXRbgkXk5lPgcrA9ZKazHujSFVU1OiQ6klhljbrnI964Dpg1/j/hjw4WAd40xfcBRETmIFSS2X+RnKTUqBV4Hd88Kcqijlw11YV6s6WROtoubSv3kuO2pbp5S4ybRnsLbIrLoIt97OzBbRGaIiAu4B1h31j6/xOolICIFWMNJRy7yc5QaEyLCnBw3X6vKZWWJj6NdvfxoXxtbGsL0DWqWkpocEu0pXAs8KCJHgR5AAGOMWXy+Fxhj+kXk68DrWPMFTxlj9ojId4Edxph18eduFpG9wADwl8aY1lEcj1Kj5rAJV0/xsTDPzab6CFsbo3zc2sONpX7m5bi0Cqua0CSRPG0RmX6ux40xx8e8RSOorq42O3bsuPgX1jbCkRBcuxTsOhygElfb3ccboW6aogNMCzhYUxagyKsprCqziMj7xpjqkfZLaPgofvLPAf4g/pWTioCgVCpMCzh5cG4On5nmpyU6wNP72/ldbTfRfk1hVRNPQkFBRP4c+ClQFP96TkT+NJkNUyqd2ERYWuDl0fm5LCv0sLMlxr/vbeOD5iiDele0mkAS7QM/Aiw3xoQBROTvgXeAf0lWw5RKR16HjTVlAS7Lt+6K/l0ozM6WGGvKApRnOVPdPKVGLdHsI8GaCD5tIP6YUpNSkdfB2sogn6/IomfA8HxNB7882klHr1ZhVZkt0Z7C08C7IvIf8e3PA08mp0lKZQYRYV6um1nZLt49GWXbyQg1Hb1cVexjebEXp02vm1TmSSgoGGO+LyKbsFJTAR4yxuxMWquUyiBOm3BtiY9F+W421oV5qzHCrlMxbir1MzdbU1hVZrlgUBCRoDGmU0TygGPxr9PP5RljTiW3eUpljmyXnc/PCHK8q5f1oTC/PNrF9ICT1WV+CjWFVWWIkX5TnwduA94HhqdYSHx7ZpLapVTGmp7l4qF5Tna2xNjSEOGp/e0sK/Rw3RQfHocudqjS2wWDgjHmtvi/M8anOUpNDDYRLi/0UpXrZktDhA+aY+w91cP1U/0szndrFVaVthK9T+EaEfHHv79PRL4vIlrDWqkR+Bw2PjMtwINzc8j32PltbTfPHGgn1N2X6qYpdU6J9mX/FYiIyGXAXwCHgWeT1iqlJphin4N7Z2dzR0UWkX7Dc4c6WHesiy5NYVVpJtHZr35jjBGRO4D/Z4x5UkQeSWbDlJpoRISqXDezgi62nYzwblOUQx09rCj2cWWRF4emsKo0kGhQ6BKRvwLuA1aKiA3Q2zeVugQuu7Byqp/F+R7erAuzuSHCrtYYq8r8VAY1hVWlVqLDR3djlcx+xBjTiLVgzj8krVVKZZquMByuhcHEi+TluO18cWaQe2YFcdiEl4908bPDnbTE+pPYUKUuLNGb1xqB7w/bPgH8JFmNUipjDAzCsToInbS2C3IhO3BRb1ERdPFQlpMPmmO81RjhqX3tXF7o4ZoSHx67prCq8TXSzWtvGWOuFZEuznGfgjEmmNTWKZXO2rvg4DGI9kCW3+otXCK7CFcUeVmQ6+b3DWG2N8fY0xZPYc1z65CSGjcj3adwbfzfrPFpjlIZoH/AWrCpoRk8Llg8B4yBjw+N+q19ThufLc9iaYGXN0LdvHaiO16F1U+pX6fxVPIlep/CVSKSNWw7S0SWJ69ZSqWp1nbYsdsKCKXFUL0Acse+wzzF5+C+2dn8wfQA3X2DPHuwg18f76K7Txf2UcmVaPbRvwLLhm2Hz/GYUhNXXx/U1ELTKfB5YOksCF7c3MHFEhEW5HmYne3mnZMR3muKcrC9l6uneKku1BRWlRyJBgUxwxZzNsYMiohW+FITnzHQ3AY1J6xho+klUF4CtvGbAHbZJV4ew8OGujCb6iN81BpjVWmAymzXuLVDTQ6J/mYfEZE/ExFn/OvPgSPJbJhSKdfTC3sOw74j1tzBsiqoKB3XgDBcrtvOXTODfHlWEEH4xZFOfn64g1MxvStajZ1Ef7v/CLgaqANCwHLg0WQ1SqmUMsaaM9i+B9o6YGYZLK2CgC/VLQNgZtDFI/NyuKnUT213Pz/a38bGujA9AzrfoEYv0fsUmoB7ktwWpVIvGoODx6100+wAzK0AryfVrfoUu0248nQKa32Yd5ui7D4V44apfhZqCqsahUSzj+aIyAYR2R3fXiwi/yu5TVNqHBkDtY2wY691v8Hs6XDZ3LQMCMP5nTZunZ7F/XOyyXbZ+c2Jbp492EFDWKuwqkuT6PDRE8BfAX0AxphdaM9BTRThKOzcb917kJMFVyyEqYWQQVfbU/1Ovjonm8+VB+joHeCZgx385ngXYU1hVRcp0QwinzHmvbO6pFqgRWW2wUE40QgnGsBhh6oZUJiXUcFgOBFhUb6HOTku3m6Msr35zBRWu6awqgQkGhRaRGQW8VIXInIX0JC0VimVbJ3d1txBOApFeTBrGrgmxh3DbruNG0v9XJbvYUNdNxvrI3zU2sPqMj8zg5rCqi4s0aDwX4HHgXkiUgccBe5NWquUSpaBAThWbxWwczlhQSUU5KS6VUmR57HzpVnZ1HT0sqGum58d7qQy6GJVmZ9ctz3VzVNpasSgEF87odoYszq+JKfNGNOV/KYpNcbaOq3eQawHSgphZik4Jv49mJXZLiqyctnRHOXtxig/2tfGFUVeri724bLrkJI604h/EfG7l/878DNjzKWXgVQqVfr74wXsWsDjtgrYJaFeUTpz2ISrin0szPOwqT7MtpNRdp/q4capPubnagqr+kSi2UfrReSbIjJNRPJOf430IhG5RUQOiEiNiHzrAvvdKSJGRKoTbrlSiWhpt25Ca2iBsmKonj/pAsJwAaeN26Zn8dU52QScNn51vJvnDnXQGNG8EWVJtO98N9Yk85+c9fjM871AROzAY8AarLugt4vIOmPM3rP2ywL+HHg30UYrNaLePmsltKZT4PdacwdBf6pblTZK/U4emJPNrlM9/L4+zI8PtHNZvpvrS/z4nLqwz2SWaFCYjxUQrsUKDluAfxvhNVcCNcaYIwAi8iJwB7D3rP3+N/D3wF8m2Balzs8YKxDU1FqTytOnQvmUlNUrSmciwmX5HubmuNjaEOH95hj723u5doqPZYUe7DqkNCkl+pfyDFAF/AD4F6wg8cwIrykFaodth+KPDRGRZcA0Y8xvEmyHUucX64XdNbD/KHjdcPl8qJiqAWEEHruNVWUBHq7KYarPwYa6ME/vb+dYZ2+qm6ZSINGewkJjzPxh2xtF5Owr/osSz2r6PvBgAvs+SrwAX3l5+Wg+Vk1ExlhzBkdqrX7srDJrARy90r0oBR4HX54V5FBHL2/WhXnxcCdzsl3cVOonR1NYJ41EL6E+EJGrTm/EV13bMcJr6oBpw7bL4o+dlgUsBDaJyDHgKmDduSabjTGPG2OqjTHVhYWFCTZZTQqRGHx0AA4dt9ZJrl4AZVM0IFwiEWFOjps/rMplZYmPo129PLGvjc31YXoHzMhvoDJeoj2Fy4G3ReREfLscOCAiHwPGGLP4HK/ZDswWkRlYweAe4CunnzTGdAAFp7dFZBPwTWPMSMFGKat3EDoJx+pAbDBnOkwp0GAwRhw24eopPhbmudlUH+Htk1E+PtXDjaV+qnJcmsI6gSUaFG652Dc2xvSLyNeB1wE78JQxZo+IfBfYYYxZd7HvqRQA3RE4eAy6IpCfA7PLwa3lG5Ih6LJze0UWSws8rA91s+5YFx/4HawpC1Dsm/g3/k1Gia6ncPxS3twY8yrw6lmP/fV59r3hUj5DTSKDg3C8wSpx7bBD1UwozNXewTiYFnDywNwcdrV+ksK6pMDDdSU+fA6dyJ9INNSrzNDZDQeOWXMIRXlQWQ5O/fUdTzYRlhR4mJfjYktjhA+aY+xr6+G6Eh9LCzzYNDhPCPpXpdLbwAAcrYO6JnA7YWGlNWSkUsbjsLGmLMCSfA/rQ2HeCIX5sCXG6jI/07N0GC/TaVBQ6aut05o7iPVai97MKLOGjVRaKPQ6uKcyyMGOXjbUhXmhppO5OVYKa7ZL/58ylQYFlX76++FwCBpbrJvQLptrrYim0o6IMDfHzcygi/eaorzTGOFwRy/Li71cVezDqQv7ZBwNCiq9tLTBoRNW7aJpU6wyFXadyEx3TptwTTyFdWNdmK2NUT5u7eGmUj9zNYU1o2hQUOmhtw9qTkBzm1XAbmGldTOayijZLjufnxHkRFcfb4S6+eWxLsoDTlaX+Sny6ukmE+j/kkqtoQJ2J2Bg0KpVNE0L2GW68iwnD83L4cOWGJsbIjy9v52l8RRWr6awpjUNCip1Yj1WeYpTnVZZ6zkVVi9BTQg2EZYVeqnKdbOlIcLOFiuFdeVUH5flawprutKgoMafMVDfDEdD8QJ206C0SG9Cm6C8Dhs3TwuwpMDDG6FuXq8Ns7MlxuqyAOUBZ6qbp86iQUGNr0jMSjPt6LYyiuZUWBlGasIr8jr4SmU2+9t72VgX5vlDHVTluLix1E9QU1jThgYFNT6MscpTHKu3sonmVkBxvvYOJhkRoSrXTWW2i20nI7x7MkpNZy9XFftYXuTFoSmsKadBQSVfd8QqUdEdgYIcq0SFFrCb1Jw24boSP4vyPGysD7OlIcKu1hg3lfqZk60prKmkQUElz+AgHK+HE41WnaL5s6wCdkrF5bjtfGFGkGNdvawPhfmPo11UZDlZXeqnQFNYU0J/6io5OrqtuYNIzBommjVNC9ip86rIcvHwPCcftMTY0hDhyf3tXF7o4dopPjyawjqu9K9Uja0zCti5YNFsyMtOdatUBrCJUF3oZX6Om80NEXY0x9jb1sP1JX4W5bs1hXWcaFBQY+dUBxw8Dj0Fzlg1AAASVUlEQVS9MLUIZpRqATt10XxOG7eUWyms60PdvFbbHU9h9VOmKaxJp0FBjV5fPxyuhZOt4PXAkrmQrQXs1OhM8Tm4d3Y2+9p62Vgf5rlDHSzIdXNDqY8sp15sJIsGBTU6zW3WXcl9/VZ5ioqpWqJCjRkRYX7esBTWpigHO3q4utjHFZrCmhQaFNSl6e2zqpm2tEHAC4vmQJYv1a1SE5TLLqyc6mdRvoc368L8viHCR60xVpX5qQxqCutY0qCgLo4x1jDR4VqrgN2MUigr1t6BGhe5bjt3zgxytNNKYX35SBczs5ysKvOT79HT2VjQn6JKXKzHmkhu64RgAOZOB58WsFPjb0bQxcNVTj5ojvFWQ4Qn97VTXeTlmile3Lr+xqhoUFAjMwbqm+BInbVdWW4tj6lddpVCdhGuKPIyP9fN7xvCvNcUZc+pGNdP9bMoz61DSpdIg4K6sEgUDhyHzm7IDcKc6eDRAnYqffidNm4tz2JpgYf1oTCvnrBSWNeU+Znq1xTWi6VBQZ3b4CDUnrTKVGgBO5UBSnxO7pudzZ62HjbVRfjJwQ4W5rm5YaqfgFOHlBKlQUF9WlfYKlHRHYWCXJhdDi694lLpT0RYmOdhdraLdxqjvNcc5WB7L9dM8VJd6MWuKawj0qCgPjEQL2BX22gFAS1gpzKU227jhlI/i/M9bKjrZmN9hI9ae1hV6mdWtlbovRANCsrS0WWVt472wJR8mKkF7FTmy/PY+dKsbA539LK+rpufH+lkVtDJqtIAeR69K/pc9K9+susfsJbFrG8GjxawUxPTrGwXFVm57GiOsrUxyo/2t3FloZcVmsL6KRoUJrPhBexK4wXs7Hr1pCYmu01YXuxjQZ6HTfVhtjVF2X2qhxtKfSzI1RTW0zQoTEbDC9j5PLBkHmQHUt0qpcZFwGnjtulZLCvw8EYozK+Pf1KFtcSnCRUaFCYTY6xaRYdOWMNG5SUwvURLVKhJaarfyf1zsvn4VA+b6sM8c6CDxfluri/x45/EKaxJPXIRuUVEDohIjYh86xzPf0NE9orILhHZICLTk9meSa2nF/Ychr1HrMVvllVZw0UaENQkJiIszvfw6Pxcrizysru1h8f3tvFeU5QBY1LdvJRIWk9BROzAY8AaIARsF5F1xpi9w3bbCVQbYyIi8sfA/wXuTlabJiVjoDFewM7EC9hNm6I3oSk1jMdu46ZSP5flu1kfCvNmXZiPWmOsLvUzIzi5UliTeZl4JVBjjDlijOkFXgTuGL6DMWajMSYS39wGlCWxPZNPtAd2HbRuRAt44fIF1pCRBgSlzinf4+DLs4LcOTOLgUHDS4c7eflIJ+09A6lu2rhJ5pxCKVA7bDsELL/A/o8Ar53rCRF5FHgUoLy8fKzaN3EZY62RfLQOBOuO5BItYKdUIkSE2dluZmS52N4U5e2TEZ7Y18uVRV5WFPtw2Sf231FaTDSLyH1ANXD9uZ43xjwOPA5QXV09OQf6EhWOWjehdYUhLwizK6z7D5RSF8VhE1ZM8bEwz82m+gjvnIynsE71MX8Cp7AmMyjUAdOGbZfFHzuDiKwG/idwvTGmJ4ntmdgGB63yFMcbrHsN5s2AojztHSg1SlkuO39QYVVhfSPUza+GUlgDTPGlxXX1mErmEW0HZovIDKxgcA/wleE7iMhS4N+BW4wxTUlsy8TWFbZ6B+GoVauoUgvYKTXWygJOHpibw8etPWxqCPPjA+0syfewssSHbwKlsCYtKBhj+kXk68DrgB14yhizR0S+C+wwxqwD/gEIAD+Pd8VOGGNuT1abJpyBQThWB6GTVhBYMMuqaqqUSgqbCJcVeJib4+KtxgjvN8fY197DdSU+lhV4sE2AnnlS+z7GmFeBV8967K+Hfb86mZ8/obV3WVlF0R6YUgCzysAx8bqySqUjj8PG6rIAS/I9rK8Lsz4U5sP4XdEVWZk9h6dnkUzTPwBHQtDQbK2AtniOtSKaUmrcFXgd3D0ryKGOXjbUhXmxppM52S5uKvWT487MOmIaFDJJazscOg49fVBWDBVTtYCdUikmIszJcTMj6OK9pijbTkb40b5elhd7uarYhzPDFvbRoJAJ+vqgphaaTlkF7JbOgqAWsFMqnThtwjVTfCzKc7OxLszWxigft/ZwY6mfeTmujElh1aCQzoyB5jaoiRewm15i3ZGs9YqUSltBl507ZgRZ2t3H+lA3/3msiw8CDtaUBSjypv8pN/1bOFn19FrVTFvbIcsHcyog4Et1q5RSCSoPOHlwbg4ftcbYXB/h6f3tLC3wcF2JD68jfS/sNCikG2OgsQUOh6wCdjPLrPmDDOl6KqU+YRNhaYGXeTlutjRE2NkSY29bDytLfCxJ0xRWDQrpJBqzVkJr77IWvZlbAV5PqlullBolr8PGzdMCLCnwsD4U5nehMDtbYqwpC1CelV43mmpQSAfGWDegHau3egSzp0NJgfYOlJpgirwO1lYGOdDey5t1YZ6v6WBejosbS/1ku9Ijk1CDQqqdUcAuG+ZMtxbBUUpNSCLCvFw3s7JdvHvSSmGt6ejlqmIfy4u9KU9h1aCQKoODcKIRTjSAww5VM6BQC9gpNVk4bcK1JT4W5VsprG81Rth1KsZNpX7mZqcuhVWDQip0dltzB+GoVcl01jQtYKfUJJXtsvP5GUGOd/WyPhTml0e7mB5wsrrMT2EKUlg1KIyngQFr3mCogF0lFOSkulVKqVQwxhox6BuA/n6mD/TzUIFhZ4dhS3cvT+3vZZmjn+tsETwD1j5MK7EqISeRBoXx0tZp9Q5iPdYqaDNLtYCdUhPB4KB1c2lfv3XiHvp+4NzbfcMeN2euGWYDLgeqxMYWdw4fmCz2ksX1hFnsANs4zDfoWSnZ+vvjBexarAJ2l82BHC1gp1RaMebTJ/azT+DnO9EPDl74vR32+JfD+jfg+uR7hx2cjk+2nda/PoeDz9htLIkO8Eaom9+Gbex02bnZ46c0yT8KDQrJ1BIvYNerBeyUSjpjrCHa/oFhJ/OzTuzn2x4YuPB722zgHHZi97ghy//JyX74c2dvj2LCuNjn4N7Z2eyPp7B2940QgMaABoVk6I0XsGs+BX6vNXcQ9Ke6VUqlv9Pj7CMNwZxvSOZCRM64GsfltP4+h1/FO89zYk9hvTERoSrXTWW2C8c4JCRpUBhLxliVTGtqrSuP6VOhfIoWsFOTz+kT+0hX6uc60Z81zv4pQyfs+Enb4z7zBD78xD5822bL6JTv8bp/QYPCWIn1WkNFpzqsbuXcCusqRKlMdXqcfaTJ0v7+Yc/F9xtpnN1uP/OE7fNeeAjm9LZ9dMMxamQaFEbLGGsS+UgtGKxlMUu1gJ1KE8ZYa3knNAQz7MR++vkLsdnOPLF73Oc+oZ9rSEb/PtKWBoXRiMSsdZI7uiEnyypv7XWnulVqIhocPP+V+hkn+rOv4j+d9niG0+PsjmHj7D7PyJOnTocOi05QGhQuxVABuzoQm1WvaIoWsFMjMGaEIZjzDcn0w+BI4+xnnbg9rgtfqQ8/sevvrRpGg8LF6o5YvYOuCOTnwOxyLWCnPnEkZF1pn+vKfWCkcXbbmVfmp6/Yzzt5OizXXU/saoxoUEjU4CAcb4DaxngBu5nW7eb6x6jAujAQsepaxXo+OYG7XeC/0OTpsBO7DseoNKBBIRGd3VZ560jMKmBXWW79MSt1mt8L1y3TiwSV8fTMdiEDA3C0DuqawO2EhZXWkJFS56IBQU0AGhTOp63TmjuI9cLUQphRZnXxlVJqAtOgcLb+fjgcgsYWK730srlWuqlSSk0CGhSGa2mDQyes2kXTplhlKuw6+aeUmjw0KEC8gN0JaG6zJgwXVlqlKpRSapKZ3EFhqIDdCSuHvGKq1UPQ1ECl1CSV1LOfiNwiIgdEpEZEvnWO590i8lL8+XdFpCKZ7TlDrAd2H4L9R62bhC6fbw0XaUBQSk1iSespiIgdeAxYA4SA7SKyzhizd9hujwBtxphKEbkH+Hvg7mS1CbCK1tU1wdFQvIDdNCgt0nRCpZQiuT2FK4EaY8wRY0wv8CJwx1n73AE8E//+F8AqkSSfnXcdtIaLggG4YoG1IpoGBKWUApI7p1AK1A7bDgHLz7ePMaZfRDqAfKAlaa2Kxqy1DorzNRgopdRZMmKiWUQeBR4FKC8vv7Q3Kcixqk6WFmkBO6WUOo9kDh/VAdOGbZfFHzvnPiLiALKB1rPfyBjzuDGm2hhTXVhYeGmt8XpgZpkGBKWUuoBkBoXtwGwRmSEiLuAeYN1Z+6wDHoh/fxfwpjEjLdCqlFIqWZI2fBSfI/g68DpgB54yxuwRke8CO4wx64AngWdFpAY4hRU4lFJKpUhS5xSMMa8Cr5712F8P+z4GfCmZbVBKKZU4vVNLKaXUEA0KSimlhmhQUEopNUSDglJKqSEaFJRSSg2RTLstQESageOX+PICkllCIz3pMU8OesyTw2iOeboxZsS7fzMuKIyGiOwwxlSnuh3jSY95ctBjnhzG45h1+EgppdQQDQpKKaWGTLag8HiqG5ACesyTgx7z5JD0Y55UcwpKKaUubLL1FJRSSl3AhAwKInKLiBwQkRoR+dY5nneLyEvx598VkYrxb+XYSuCYvyEie0Vkl4hsEJHpqWjnWBrpmIftd6eIGBHJ+EyVRI5ZRL4c/7/eIyLPj3cbx1oCv9vlIrJRRHbGf79vTUU7x4qIPCUiTSKy+zzPi4j8IP7z2CUiy8a0AcaYCfWFVab7MDATcAEfAfPP2udPgH+Lf38P8FKq2z0Ox3wj4It//8eT4Zjj+2UBm4FtQHWq2z0O/8+zgZ1Abny7KNXtHodjfhz44/j384FjqW73KI95JbAM2H2e528FXgMEuAp4dyw/fyL2FK4EaowxR4wxvcCLwB1n7XMH8Ez8+18Aq0QyesHmEY/ZGLPRGBOJb27DWgkvkyXy/wzwv4G/B2Lj2bgkSeSYvwY8ZoxpAzDGNI1zG8daIsdsgGD8+2ygfhzbN+aMMZux1pc5nzuAnxjLNiBHRErG6vMnYlAoBWqHbYfij51zH2NMP9AB5I9L65IjkWMe7hGsK41MNuIxx7vV04wxvxnPhiVRIv/Pc4A5IrJVRLaJyC3j1rrkSOSYvwPcJyIhrPVb/nR8mpYyF/v3flGSusiOSj8ich9QDVyf6rYkk4jYgO8DD6a4KePNgTWEdANWb3CziCwyxrSntFXJtRb4sTHmH0VkBdZqjguNMYOpblgmmog9hTpg2rDtsvhj59xHRBxYXc7WcWldciRyzIjIauB/ArcbY3rGqW3JMtIxZwELgU0icgxr7HVdhk82J/L/HALWGWP6jDFHgYNYQSJTJXLMjwA/AzDGvAN4sGoETVQJ/b1fqokYFLYDs0Vkhoi4sCaS1521zzrggfj3dwFvmvgMToYa8ZhFZCnw71gBIdPHmWGEYzbGdBhjCowxFcaYCqx5lNuNMTtS09wxkcjv9i+xegmISAHWcNKR8WzkGEvkmE8AqwBEpAorKDSPayvH1zrg/ngW0lVAhzGmYazefMINHxlj+kXk68DrWJkLTxlj9ojId4Edxph1wJNYXcwarAmde1LX4tFL8Jj/AQgAP4/PqZ8wxtyeskaPUoLHPKEkeMyvAzeLyF5gAPhLY0zG9oITPOa/AJ4Qkf+GNen8YCZf5InIC1iBvSA+T/JtwAlgjPk3rHmTW4EaIAI8NKafn8E/O6WUUmNsIg4fKaWUukQaFJRSSg3RoKCUUmqIBgWllFJDNCgopZQaokFBqXEkIhWnq1+KyA0i8utUt0mp4TQoKJWA+I1C+veiJjz9JVfqPOJX9QdE5CfAbuCrIvKOiHwgIj8XkUB8vytE5G0R+UhE3hORrPhrt8T3/UBErk7t0SiVmAl3R7NSY2w2VkmUGuAVYLUxJiwi/wP4hoj8H+Al4G5jzHYRCQJRoAlYY4yJichs4AWsQoRKpTUNCkpd2HFjzDYRuQ1rAZet8TIhLuAdYC7QYIzZDmCM6QQQET/w/0RkCVa5iTmpaLxSF0uDglIXFo7/K8Abxpi1w58UkUXned1/A04Cl2EN006ERX7UJKBzCkolZhtwjYhUgtUTEJE5wAGgRESuiD+eNawce0O8pv9XsYq5KZX2NCgolQBjTDPWgj0viMgurKGjefElIu8G/kVEPgLewCrd/EPggfhj8/ikx6FUWtMqqUoppYZoT0EppdQQDQpKKaWGaFBQSik1RIOCUkqpIRoUlFJKDdGgoJRSaogGBaWUUkM0KCillBry/wEYMY492EdtVwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x106626ba8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(recall_1, precision_1, \"pink\", label=\"Model_1\")\n",
    "plt.plot(recall_2, precision_2, \"skyblue\", label=\"Model_2\")\n",
    "plt.xlabel('recall')\n",
    "plt.ylabel('precision')\n",
    "plt.legend()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use [step_plot](http://scikit-learn.org/stable/auto_examples/model_selection/plot_precision_recall.html#plot-the-precision-recall-curve) to plot PR-curves\n",
    "\n",
    "This code is found on the sklearn documentation.\n",
    "\n",
    "Especially, we notice we are using the `post` argument. From the [`step()`](https://matplotlib.org/api/_as_gen/matplotlib.pyplot.step.html) documentation:\n",
    "\n",
    "> 'post': The y value is continued constantly to the right from every x position, i.e. the interval `[[x[i], x[i+1])` has the value `y[i]`.\n",
    "\n",
    "Because we only have two points in order: $\\left(1, 0.1\\right), \\left(0, 1\\right)$, the interval between $x \\in [0, 1)$ will have the same y value $0.1$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEWCAYAAABbgYH9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XucXVV5//HP1yQwQBICCYhkIAEMSAKRQLiIclEQAW1AaC2RWySAWi1avNRfa0tAhKpFxYpVKhTkHpTaKCC1QIhQUUITIoEGEQOZAJIM15AEEnh+f+w1zM5hzpyTMOucM8P3/XrNa87Ze529n7325dl7rXP2VkRgZmbW197S7ADMzGxgcoIxM7MsnGDMzCwLJxgzM8vCCcbMzLJwgjEzsyycYMzMLIsBkWAkLZR0cI0y20taIWlQg8LKTtJiSYem1zMkXdnsmN5MJI2VFJIG11F2mqQ7GxGX9T1Jl0k6t86yr+2Xb3ZZE0yq6FXpwP6ntJKG9vV8ImJCRMyuUeaxiBgaEa/09fzTwX1NWs5nJf2PpHf19XzeLNJ2slbS2yqGD/h6lvQVSb9Lyz/jDUzno5IelfSipJ9K2rJKuVGS7pLUmer015LeXVHmbyQ9Kel5SZdK2rhi/Gck/THN60FJO69nrH22vlMiD0nfqhh+VBp+2frEloukkyXdm+q0Q9LX6zlR6WE6W0r6j1T3j0r6aC9l3yvpdknPSVrcw/ixafxKSf9XmSRrbQc9acQVzJ9FxFBgT2Ay8OXKAir096up69JyjgJuB65vcjx9bkN2gA2Yx2bAscBzwAk9FOmq562AO4EbJCl3XA30MPBF4MYNnYCkCcAPgBOBtwIrge9VKb4COIWiPrcAvgb8rGtdS/oA8CXgEGAMsCNwdmlepwLTgQ8CQ4EPAcvXI9Yc6/sPwEcqtteTgYfqjasBNgU+S3G82Jeifj+/AdO5CHiZYj0fD/xrWv89eRG4FPhClfHXAPOAkcDfAz+WtBXU3g6qadhBPSKWAjcDuwFImi3pq5LuotgBdpS0uaRLJD0haamkc1Vq0pJ0WjpDekHSA5L2TMPLTUX7SJqbsuyfJH0zDV+nOUPStpJmSXpa0sOSTivNZ4akmZJ+lOa1UNLkOpdzLXAVMLpr5aRpfkjS/NKZ2MTSuO0k3SBpWTqT/G4avpOk29Kw5ZKukjRiQ+o/ncHNT/XyB0mHV9ZdadmvrKiz6ZIeA26TdLOkT1dM+z5Jx6TX75D0y1SviyR9ZD1DPRZ4FjiH4qDQo4hYA1wObEOxQ1Rb7mkqztC/ler+EUn7p+FLJD0l6eRS+c3Tel+Wzgi/3HXyI2mQpH9O6+IRioMqFZ+tuv3WIyIuj4ibgRfW53MVjgd+FhFzImIF8A/AMZKG9TC/1RGxKCJeBQS8QpFouq54TgYuiYiFEfEM8BVgGkCql7OAv4mIB6Lwh4h4ej1i7dP1nTwJ/A74QIpzS2B/YFa5kKQpad9+Nh2Pdi2NmyTpf9P+fx3QVvHZqvtzPSLiXyPiVxHxcjo2XgW8u9bnKmLoSs7/EBErIuLOtIwnVpnnbyPiCuCRHqa1M8VFwFkRsSoifkJRh8emIlW3g940LMFI2g44kiJDdjkROB0YBjwKXAasBd4OTAIOA05Nn/8LYAZwEjAcmAJ09jCrC4ELI2I4sBMws0pI1wIdwLbAnwPnSXpfafyUVGYExUr7bp3LuVGKsRN4Jg2bRHHm8HGKneMHwCxJG6cD0M/T8o8FRqf5QrHDn59i3BXYLtXBepG0D/AjijOXEcCBwOL1mMRBaf4foDjLmVqa9niKM5ob0wb/S+BqYGvgOOB7qUxXs82CGvM6Oc3jWuAdkvaqskwbU2zgSyKi1hnzvsACirq/Ok17b4rt7ATgu+puuv0XYHOKM7SDKNblx9K40yjO0CdRXI3/ecV8LqPK9tsXJL0nHdCq/b0nFZ0A3Nf1uYj4A8VZbtWmq7ReVlNs6z+MiKd6mlZ6/VZJI4H29LdbStZ/lHS21q81Isf6hmJ7Pym9Pg74T+Cl0vR2TvP9LMXV0U0UV24bpX34p8AVFIn2eroPtL3uz/Utco8OBBaW5vHzXtbzz1OxnYG1EVG+MruPYp2trwnAIxFRPrEpT6u37aC6iMj2R3EQW0FxhvIoxWX6JmncbOCcUtm3UmwAm5SGTQVuT69vAT7Ty3wOTa/nUFy6jaooMxYIYDDFgfoVYFhp/PnAZen1DOC/S+PGA6t6Wc4ZFDvws2m6ncDBpfH/Cnyl4jOLKA5g7wKWAYPrqM+jgXlVlnsGcGWVz/0A+FatuqucTqnOdiyNH0ZxqT0mvf8qcGl6/ZfAr3qY91l1bi/bA68Ce5TW+YVV6vkp4DZgrxrTnAb8vvR+97RMby0N6wT2AAal6Y8vjfs4MDu9vg34RGncYaVtqtb2Ow24cz32nSuBGRu4391ajjMNW1reJqt8ri3FfHJp2B+Aw0vvh6RlHktxVRAUzXkj0rCHgNOavL7vBDYB/kRxsnA3xdXBuXTv4/8AzCx97i1ddURxsH8cUGn8/wDnRo39uad9qo56OIXiZHdUvZ9JnzsAeLJi2Gld22svnzsUWFwx7ETg7ophXy3VV9XtoLd5NeIK5uiIGBERYyLiryJiVWncktLrMSnoJ7oyNcXBaes0fjuKhaxlOkVm/z9J90j6UA9ltgWejnWz9aMUVw9dniy9Xgm0SRos6XgVnY4rJN1cKjMzIkZQHGjuB8pnYmOAz5XPQtLybJv+PxpF09o6JL1V0rWpueV5ioPOqDrqoFK9dVfNa+sp1dmNFGeFUByQrkqvxwD7Vizn8RTNGvU4EXgwIuan91cBH5U0pFRmZtqeto6I90XEvXVM90+l16vSclQO6+o/G0KxLXQpbxfbsu42Wy5Xa/ttpBUUV/llw6nR7BZFc9k1wJckvbPKtLpev0CqS+DrEfFsRCymWOYj64wz1/omHWdupOjzHRkRd1UU2ZbS+ouiiXAJxbreFlga6UiaVK7ravvzepF0NMXJ7RFR35VZ2Qat5w2cVm/bQVXN7lgvr8AlFGeAo9IGNSIihkfEhNL4nWpOMOL3ETGVYsf+GkVH1WYVxR4Htqxok96e4gym1vSviuLbaEMj4ogexi+naPaboe5vxSwBvlparhERsWnamZcA26vnDvTzKOpo9yia/E6gaDZbX73V3YsUHY5dekoGlc90uAaYquIbPW0UX2roms8dFcs5NCI+WWecJ1H0xT0p6UngmxQH/XoPWG/UcmANxQGkS3m7eILiQFIe16XW9vuGSTqgdHLT098BqehC4J2lz+0IbEz9ndxDKJoIXzet9PpPEdFJcdb+MutuH+vz/I/c6/tHwOcoTswqPU5pPUsSxbpdSrGeR6dhXSrXdbX9uW4q+kH/jeKLUL+rGHdzL+u568T2IWCwpHGlj76TUlPbelhIsS7Kx8TytHrbDqpqdoJ5TUQ8AfwXcIGk4ZLeoqKT+6BU5IfA5yXtpcLbJY2pnI6kEyRtlc5Ink2DX62Y1xKKS97zJbWlDrrp9LwhbsiyLKK43P9iGvRvwCck7Zti30zSB9PK/C3FBv1PaXibur8mOozizOE5SaOp/u2PWi4BPibpkFSvoyW9I42bDxwnaYiKLzJU9iv05CaKnfMcim/5dNXvz4GdJZ2YpjdE0t4qdZ5Wk5LVTsA+FM1Ve1B8IeRqutvSs4riK+wzga9KGpa2rzPp3i5mAmdIape0BcW3aro+W2v7rUuqszaKfXNw2h4GpXn8qnRy09Pfr9JkrgL+LCWkzSjW0w0VV+xd89tPRd/ORpI2kfS3FFfhv0lFfgRMlzRexRdMvkzR10RErASuA76Y6qud4uTq52naXV8SGdvDfBuxvu8A3k/Rr1ZpJvDBtE8MoUhEL1EcF35N0Zd2Rlofx6Q4u/S2P9dFRX/vVcCxEfHbyvERcUQv6/mIVOZF4AbgnBTDu4GjKPqOeprnW9K2NaR4qzYV/U1E0Y8zHzgrDf8wMBH4Sfp41e2gV/W09W3oH720RVL0wZxaMWxzivbNDoqvLc4DjiuN/wTFWdMKimaoSZXzoTgYPJXKLKRoooNSH0x6306xIzxN0XxUblufQak/o/KzPSzLOuXTsH0prg62Tu8PB+6hSHpPUHQcDovutuifUvQFLAe+k4ZPAO5NyzKfYifo6Kl+e4qhIp4PU3R0v0DxVdgPpOE7UhxMVlA0KXyH1/fBvG65KZJWAHtXDN8lTWdZWp7b6G5jPx5YWCW+7wM/6WH4PhQ7/pa1lrHKdKdR6vug6ICPijIdwHvS6y3SNrSM4kz1H4G3pHGDgW+l5foj8KmKbarq9lsZRy/xXpamWf6btgH73keBx9I2+J/AlqVxNwN/l14fRNFh+wLFvnAHcGDFtM6kaGZ8Hvh3YOPSuOEUHfQvlOpLadwBaRsd0qz1XTHutT6Y0j7xQFpXdwATSuMmp/X3AkUSvY7UB1PH/ryYGn0wFFf9ayn2u66/mzdgPW9Jcex4Ma3vj5bGHQCsKL0/uIdta3Zp/FiK4/IqiuPsoRXzqrodVPvr2hDMzPqUpC8DyyLiB82OxZrDCcbMzLJomT4Ysw0l6ftVOkO/3+zYKvXWUd/s2PqL/rK+e+mo/7tmx9YovoIxM7Msst9bqq+NGjUqxo4d2+wwzMz6lXvvvXd5RGxVu2Tf6XcJZuzYscydO7fZYZiZ9SuSHq1dqm+5D8bMzLJwgjEzsyycYMzMLIt+1wdjZm8ua9asoaOjg9WrVzc7lH6hra2N9vZ2hgwZUrtwZk4wZtbSOjo6GDZsGGPHjkUD6uGlfS8i6OzspKOjgx122KHZ4biJzMxa2+rVqxk5cqSTSx0kMXLkyJa52suWYCRdquJxtPdXGS9J31HxuOIFSo8/NjOr5ORSv1aqq5xXMJdR3HG0miOAcenvdIq70JqZ2QCRrQ8mIub09ByIkqOAH0Vxr5q7JY2Q9LYonqtR1ZoXX+7DKM2sv7n7bnj22drl6jViBOy3X99Nz7o1sw9mNOs+fraDdR9Z/BpJp0uaK2nu8qXLGhKcmbWmZ5+Frbbqu796kpUkTjjhhNfer127lq222ooPfainJ7JXN3bsWJYv7/3JyLXKnHLKKWy99dbstttu6zXvZugXnfwRcXFETI6IyUM3HdrscMzsTWazzTbj/vvvZ9WqVQD88pe/ZPToHs+Hs5s2bRq/+MUvmjLv9dXMBLOUdZ9v3k73s8/NzFrKkUceyY033gjANddcw9SpU18b9/TTT3P00UczceJE9ttvPxYsWABAZ2cnhx12GBMmTODUU0+lfPf6K6+8kn322Yc99tiDj3/847zyyit1xXHggQey5ZZb9uGS5dPMBDMLOCl9m2w/4Lla/S9mZs1y3HHHce2117J69WoWLFjAvvvu+9q4s846i0mTJrFgwQLOO+88TjrpJADOPvts3vOe97Bw4UI+/OEP89hjjwHw4IMPct1113HXXXcxf/58Bg0axFVXXdWU5copWye/pGsongE9SlIHcBYwBCAivg/cBBxJ8Xz4lcDHcsViZvZGTZw4kcWLF3PNNddw5JFHrjPuzjvv5Cc/+QkA73vf++js7OT5559nzpw53HDDDQB88IMfZIsttgDg1ltv5d5772XvvfcGYNWqVWy99dYNXJrGyPktsqk1xgfwqVzzNzPra1OmTOHzn/88s2fPprOzc4OnExGcfPLJnH/++X0YXevpF538ZmZdRoyAZcv67m/EiPrnfcopp3DWWWex++67rzP8gAMOeK2Ja/bs2YwaNYrhw4dz4IEHcvXVVwNw880388wzzwBwyCGH8OMf/5innnoKKPpwHn204Y9ryc73IjOzfqWZv1lpb2/njDPOeN3wGTNmcMoppzBx4kQ23XRTLr/8cqDom5k6dSoTJkxg//33Z/vttwdg/PjxnHvuuRx22GG8+uqrDBkyhIsuuogxY8bUjGHq1KnMnj2b5cuX097eztlnn8306dP7dkH7iMrfaugPdtlmXCx68vfNDsPMGuTBBx9k1113bXYY/UpPdSbp3oiY3Mg43ERmZmZZuInMzKzFdHZ2csghh7xu+K233srIkSObENGGcYIxM2sxI0eOZP78+c0O4w1zE5mZmWXhBGNmZlk4wZiZWRbugzGz/sUPhOk3fAVjZv1LEx4I0yrPg1myZAnvfe97GT9+PBMmTODCCy9cr/k3mq9gzMxqKD8PZpNNNmna82AGDx7MBRdcwJ577skLL7zAXnvtxfvf/37Gjx/f8Fjq4SsYM7M6tMLzYN72trex5557AjBs2DB23XVXli5t3cdoOcGYmdWh1Z4Hs3jxYubNm7dOHK3GTWRmZnVopefBrFixgmOPPZZvf/vbDB8+vC8WLwsnGDOzOrXC82DWrFnDsccey/HHH88xxxyzwTE0gpvIzKx/aeIDYZr9PJiIYPr06ey6666ceeaZdcfdLL6CMbP+pYm/WWn282DuuusurrjiCnbffXf22GMPAM4777zXNdm1Cj8Pxsxamp8Hs/78PBgzMxvQ3ERmZtZi/DwYM7MGiQgkNTuMhnkjz4NppW4PN5GZWUtra2ujs7OzpQ6crSoi6OzspK2trdmhAL6CMbMW197eTkdHB8uWLWt2KP1CW1sb7e3tzQ4DcIIxsxY3ZMgQdthhh2aHYRvATWRmZpaFE4yZmWXhBGNmZlk4wZiZWRZOMGZmloUTjJmZZeEEY2ZmWWRNMJIOl7RI0sOSvtTD+O0l3S5pnqQFklrzntNmZrbesiUYSYOAi4AjgPHAVEnjK4p9GZgZEZOA44Dv5YrHzMwaK+cVzD7AwxHxSES8DFwLHFVRJoCuB0pvDjyeMR4zM2ugnAlmNLCk9L4jDSubAZwgqQO4CfjrniYk6XRJcyXNXbFyRY5YzcysjzW7k38qcFlEtANHAldIel1MEXFxREyOiMlDNx3a8CDNzGz95UwwS4HtSu/b07Cy6cBMgIj4NdAGjMoYk5mZNUjOBHMPME7SDpI2oujEn1VR5jHgEABJu1IkGN+T28xsAMiWYCJiLfBp4BbgQYpviy2UdI6kKanY54DTJN0HXANMCz9VyMxsQFB/O57vss24WPTk75sdhplZvyLp3oiY3Mh5NruT38zMBignGDMzy8IJxszMsnCCMTOzLJxgzMwsCycYMzPLwgnGzMyycIIxM7MsnGDMzCwLJxgzM8vCCcbMzLJwgjEzsyycYMzMLAsnGDMzy8IJxszMsnCCMTOzLJxgzMwsCycYMzPLwgnGzMyycIIxM7MsnGDMzCwLJxgzM8vCCcbMzLJwgjEzsyycYMzMLAsnGDMzy8IJxszMsnCCMTOzLJxgzMwsCycYMzPLwgnGzMyyGFxvQUmjgTHlz0TEnBxBmZlZ/1dXgpH0NeAvgQeAV9LgAHpNMJIOBy4EBgE/jIh/6qHMR4AZaXr3RcRH6w3ezMxaV71XMEcDu0TES/VOWNIg4CLg/UAHcI+kWRHxQKnMOOD/Ae+OiGckbV1/6GZm1srq7YN5BBiyntPeB3g4Ih6JiJeBa4GjKsqcBlwUEc8ARMRT6zkPMzNrUfVewawE5ku6FXjtKiYizujlM6OBJaX3HcC+FWV2BpB0F0Uz2oyI+EWdMZmZWQurN8HMSn855j8OOBhoB+ZI2j0ini0XknQ6cDrAtsO2yRCGmZn1tboSTERcLmkj0hUHsCgi1tT42FJgu9L79jSsrAP4TZrWHyU9RJFw7qmY/8XAxQC7bDMu6onZzMyaq64+GEkHA7+n6LT/HvCQpANrfOweYJykHVJyOo7XXwX9lOLqBUmjKBLYI/UGb2ZmraveJrILgMMiYhGApJ2Ba4C9qn0gItZK+jRwC0X/yqURsVDSOcDciJiVxh0mqevrz1+IiM4NXxwzM2sViqjd4iRpQURMrDWsEXbZZlwsevL3jZ6tmVm/JuneiJjcyHnWewUzV9IPgSvT++OBuXlCMjOzgaDeBPNJ4FNA19eSf0XRF2NmZtajer9F9hLwzfRnZmZWU68JRtLMiPiIpN9R3CtsHc3ogzEzs/6h1hXMZ9L/D+UOxMzMBpZefwcTEU+kl8uBJRHxKLAx8E7g8cyxmZlZP1bvzS7nAG3pmTD/BZwIXJYrKDMz6//qTTCKiJXAMcD3IuIvgAn5wjIzs/6u7gQj6V0Uv3+5MQ0blCckMzMbCOpNMJ+leDDYf6TbvewI3J4vLDMz6+/q/R3MHcAdpfeP0P2jSzMzs9ep9TuYb0fEZyX9jJ5/BzMlW2RmZtav1bqCuSL9/+fcgZiZ2cDSa4KJiHvTy7nAqoh4FUDSIIrfw5iZmfWo3ptd3gocCqxI7zeh+D3M/jmC6k0EPPRQo+dqVtuWW8KoUc2Owqx11Jtg2iKiK7kQESskbZoppl69GjBnTjPmbFbdSy8VCWbq1GZHYtY66k0wL0raMyL+F0DSXsCqfGFVF6/CpEnNmLNZdY8+Ck8/3ewozFpLvQnms8D1kh4HBGwD/GW2qMzMrN+r93cw90h6B7BLGrQoItbkC8vMzPq7un7Jn/pb/hb4TETcD4yV5Fv4m5lZVfXeKubfgZeBd6X3S4Fzs0RkZmYDQr0JZqeI+DqwBiDdWVnZojIzs36v3gTzsqRNSLeLkbQT8FK2qMzMrN+r91tkZwG/ALaTdBXwbmBarqDMzKz/q5lgJAn4P4qHje1H0TT2mYhYnjk2MzPrx2ommIgISTdFxO50P2zMzMysV/X2wfyvpL2zRmJmZgNKvX0w+wInSFoMvEjRTBYRMTFXYGZm1r/Vm2A+kDUKMzMbcGo90bIN+ATwduB3wCURsbYRgZmZWf9Wqw/mcmAyRXI5Argge0RmZjYg1GoiG5++PYakS4Df5g/JzMwGglpXMK/dMdlNY2Zmtj5qJZh3Sno+/b0ATOx6Len5WhOXdLikRZIelvSlXsodKykkTV7fBTAzs9bUaxNZRAza0AlLGgRcBLwf6ADukTQrIh6oKDcM+Azwmw2dl5mZtZ56f2i5IfYBHo6IRyLiZeBa4Kgeyn0F+BqwOmMsZmbWYDkTzGhgSel9Rxr2Gkl7AttFRK+3oJF0uqS5kua+uPrFvo/UzMz6XM4E0ytJbwG+CXyuVtmIuDgiJkfE5M3aNssfnJmZvWE5E8xSYLvS+/Y0rMswYDdgdroFzX7ALHf0m5kNDDkTzD3AOEk7SNoIOA6Y1TUyIp6LiFERMTYixgJ3A1MiYm7GmMzMrEGyJZj0u5lPA7cADwIzI2KhpHMkTck1XzMzaw313uxyg0TETcBNFcP+sUrZg3PGYmZmjdW0Tn4zMxvYnGDMzCwLJxgzM8vCCcbMzLJwgjEzsyycYMzMLAsnGDMzy8IJxszMsnCCMTOzLJxgzMwsCycYMzPLwgnGzMyycIIxM7MsnGDMzCwLJxgzM8vCCcbMzLJwgjEzsyycYMzMLAsnGDMzy8IJxszMsnCCMTOzLJxgzMwsCycYMzPLwgnGzMyycIIxM7MsnGDMzCwLJxgzM8vCCcbMzLJwgjEzsyycYMzMLAsnGDMzy8IJxszMssiaYCQdLmmRpIclfamH8WdKekDSAkm3ShqTMx4zM2ucbAlG0iDgIuAIYDwwVdL4imLzgMkRMRH4MfD1XPGYmVlj5byC2Qd4OCIeiYiXgWuBo8oFIuL2iFiZ3t4NtGeMx8zMGihnghkNLCm970jDqpkO3NzTCEmnS5orae6Lq1/swxDNzCyXlujkl3QCMBn4Rk/jI+LiiJgcEZM3a9usscGZmdkGGZxx2kuB7Urv29OwdUg6FPh74KCIeCljPGZm1kA5r2DuAcZJ2kHSRsBxwKxyAUmTgB8AUyLiqYyxmJlZg2VLMBGxFvg0cAvwIDAzIhZKOkfSlFTsG8BQ4HpJ8yXNqjI5MzPrZ3I2kRERNwE3VQz7x9LrQ3PO38zMmqclOvnNzGzgcYIxM7MsnGDMzCwLJxgzM8vCCcbMzLJwgjEzsyycYMzMLAsnGDMzy8IJxszMsnCCMTOzLJxgzMwsCycYMzPLIuvNLs3eLFauhFWr4KGHmh2JWetwgjHrI6tXw5w5zY7CrJphDX8ccL9LMCLY+FGfJlprGfYEPNcB++3U7EjMqhk0qNFz7HcJhoCh83yaaK1l08eHssVyGDp8RbNDMevRMNa2NXqe/S7BvDJoMKt2mdTsMMzW8dxGm7JsI1i1y8pmh2LWo0FEw7/U5W+RmZlZFk4wZmaWhROMmZll4QRjZmZZOMGYmVkWTjBmZpaFE4yZmWXR734HY9aKNtvkFTbduNlRmLUWJxizPjB2m5cYu02zozBrLW4iMzOzLJxgzMwsCycYMzPLwgnGzMyycIIxM7MsnGDMzCwLJxgzM8sia4KRdLikRZIelvSlHsZvLOm6NP43ksbmjMfMzBonW4KRNAi4CDgCGA9MlTS+oth04JmIeDvwLeBrueIxM7PGynkFsw/wcEQ8EhEvA9cCR1WUOQq4PL3+MXCIJGWMyczMGiTnrWJGA0tK7zuAfauViYi1kp4DRgLLy4UknQ6cDiC0dv8TD/pTrqD7k5dYs9nGDHmx2XG0AtdFN9dFN9dFt+dZNarR8+wX9yKLiIuBiwEkzX0mVkxuckgtQdLclfGS6wLXRZnropvropukuY2eZ84msqXAdqX37WlYj2UkDQY2BzozxmRmZg2SM8HcA4yTtIOkjYDjgFkVZWYBJ6fXfw7cFhGRMSYzM2uQbE1kqU/l08AtwCDg0ohYKOkcYG5EzAIuAa6Q9DDwNEUSquXiXDH3Q66Lbq6Lbq6Lbq6Lbg2vC/mCwczMcvAv+c3MLAsnGDMzy6JlE4xvM9Otjro4U9IDkhZIulXSmGbE2Qi16qJU7lhJIWnAfkW1nrqQ9JG0bSyUdHWjY2yUOvaR7SXdLmle2k+ObEacuUm6VNJTku6vMl6SvpPqaYGkPbMGFBEt90fxpYA/ADsCGwH3AeMryvwV8P30+jjgumbH3cS6eC+waXr9yTdzXaRyw4A5wN3A5GbH3cTtYhwwD9givd+62XE3sS4uBj6ZXo8HFjc77kx1cSCwJ3B/lfFHAjcDAvYDfpMznla9gvFtZrrVrIuIuD0iVqa3d1P85mggqme7APgKxX2wd6kpAAADmklEQVTtVjcyuAarpy5OAy6KiGcAIuKpBsfYKPXURQDD0+vNgccbGF/DRMQcim/kVnMU8KMo3A2MkPS2XPG0aoLp6TYzo6uViYi1QNdtZgaaeuqibDrFGcpAVLMu0iX/dhFxYyMDa4J6toudgZ0l3SXpbkmHNyy6xqqnLmYAJ0jqAG4C/roxobWc9T2evCH94lYxVh9JJwCTgYOaHUszSHoL8E1gWpNDaRWDKZrJDqa4qp0jafeIeLapUTXHVOCyiLhA0rsofn+3W0S82uzABrJWvYLxbWa61VMXSDoU+HtgSkS81KDYGq1WXQwDdgNmS1pM0cY8a4B29NezXXQAsyJiTUT8EXiIIuEMNPXUxXRgJkBE/BpoAxp+88cWUNfxpK+0aoLxbWa61awLSZOAH1Akl4Hazg416iIinouIURExNiLGUvRHTYmIht/krwHq2Ud+SnH1gqRRFE1mjzQyyAappy4eAw4BkLQrRYJZ1tAoW8Ms4KT0bbL9gOci4olcM2vJJrLId5uZfqfOuvgGMBS4Pn3P4bGImNK0oDOpsy7eFOqsi1uAwyQ9ALwCfCEiBtxVfp118Tng3yT9DUWH/7SBeEIq6RqKk4pRqb/pLGAIQER8n6L/6UjgYWAl8LGs8QzAOjYzsxbQqk1kZmbWzznBmJlZFk4wZmaWhROMmZll4QRjZmZZOMGYVZD0iqT5ku6X9DNJI/p4+tMkfTe9niHp8305fbNW4QRj9nqrImKPiNiN4jdWn2p2QGb9kROMWe9+TelmgJK+IOme9CyNs0vDT0rD7pN0RRr2Z+lZRfMk/bektzYhfrOmaclf8pu1AkmDKG4vckl6fxjFvbz2oXiexixJB1LcA+/LwP4RsVzSlmkSdwL7RURIOhX4IsUvys3eFJxgzF5vE0nzKa5cHgR+mYYflv7mpfdDKRLOO4HrI2I5QER0PY+jHbguPW9jI+CPjQnfrDW4iczs9VZFxB7AGIorla4+GAHnp/6ZPSLi7RFxSS/T+RfguxGxO/Bxihssmr1pOMGYVZGeEnoG8Ln0SIhbgFMkDQWQNFrS1sBtwF9IGpmGdzWRbU73rdBPxuxNxk1kZr2IiHmSFgBTI+KKdKv3X6e7Vq8ATkh37v0qcIekVyia0KZRPEXxeknPUCShHZqxDGbN4rspm5lZFm4iMzOzLJxgzMwsCycYMzPLwgnGzMyycIIxM7MsnGDMzCwLJxgzM8vi/wMbxXQBIpBJRwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10663b0f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.utils.fixes import signature\n",
    "\n",
    "# In matplotlib < 1.5, plt.fill_between does not have a 'step' argument\n",
    "step_kwargs = ({'step': 'post'}\n",
    "               if 'step' in signature(plt.fill_between).parameters\n",
    "               else {})\n",
    "\n",
    "plt.step(recall_1, precision_1, color='blue', alpha=0.2, where='post')\n",
    "plt.fill_between(recall_1, precision_1, alpha=0.2, color='blue', label=\"Model_1\", **step_kwargs)\n",
    "\n",
    "plt.step(recall_2, precision_2, color='red', alpha=0.2, where='post')\n",
    "plt.fill_between(recall_2, precision_2, alpha=0.2, color='red', label=\"Model_2\", **step_kwargs)\n",
    "\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.legend()\n",
    "plt.title('Precision-Recall curve: AP_model_1={:.3f}, AP_Model_2={:.3f}'.format(model_1_ap,\n",
    "                                                                                model_2_ap))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Computing AP using 1D Array vs. 2D Array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 20 20\n",
      "Model_1 AP = 0.30556\n",
      "Model_2 AP = 0.10000\n"
     ]
    }
   ],
   "source": [
    "model_1_score = [0.4325, 0.3498, 0.2368, 0.2601, 0.1698, 0.211 , 0.1913, 0.8441,\n",
    "                 0.098 , 0.1682, 0.1844, 0.3937, 0.1746, 0.295 , 0.3164, 0.1856,\n",
    "                 0.1353, 0.2147, 0.1898, 0.2257]\n",
    "model_2_score = [0.502, 0.502, 0.502, 0.502, 0.502, 0.502, 0.502, 0.502,\n",
    "                 0.502, 0.502, 0.502, 0.502, 0.502, 0.502, 0.502, 0.502,\n",
    "                 0.502, 0.502, 0.502, 0.502]\n",
    "y_true = [1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
    "\n",
    "print(len(model_1_score), len(model_2_score), len(y_true))\n",
    "\n",
    "model_1_ap = metrics.average_precision_score(y_true, model_1_score)\n",
    "model_2_ap = metrics.average_precision_score(y_true, model_2_score)\n",
    "print(\"Model_1 AP = {:.5f}\".format(model_1_ap))\n",
    "print(\"Model_2 AP = {:.5f}\".format(model_2_ap))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 1]\n",
      " [1 0]\n",
      " [1 0]\n",
      " [1 0]\n",
      " [1 0]]\n",
      "[[0.5675 0.4325]\n",
      " [0.6502 0.3498]\n",
      " [0.7632 0.2368]\n",
      " [0.7399 0.2601]\n",
      " [0.8302 0.1698]]\n",
      "[[0.498 0.502]\n",
      " [0.498 0.502]\n",
      " [0.498 0.502]\n",
      " [0.498 0.502]\n",
      " [0.498 0.502]]\n"
     ]
    }
   ],
   "source": [
    "y_true_2d = np.stack([[0, 1] if i else [1, 0] for i in y_true], axis=0)\n",
    "model_1_score_2d = np.stack([[1-i, i] for i in model_1_score], axis=0)\n",
    "model_2_score_2d = np.stack([[1-i, i] for i in model_2_score], axis=0)\n",
    "\n",
    "print(y_true_2d[:5])\n",
    "print(model_1_score_2d[:5])\n",
    "print(model_2_score_2d[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model_1 AP = 0.60384\n",
      "Model_2 AP = 0.50000\n"
     ]
    }
   ],
   "source": [
    "model_1_ap_2d = metrics.average_precision_score(y_true_2d, model_1_score_2d)\n",
    "model_2_ap_2d = metrics.average_precision_score(y_true_2d, model_2_score_2d)\n",
    "\n",
    "print(\"Model_1 AP = {:.5f}\".format(model_1_ap_2d))\n",
    "print(\"Model_2 AP = {:.5f}\".format(model_2_ap_2d))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Feeding `average_precision_score()` with the raw 2D probability(score) gives different AP value.\n",
    "\n",
    "The documentation says that `y_true` and `y_score` can have shape `[n_samples, n_classes]`. However, it does not mean `average_precision_score()` expects the 2D array for binary cases."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model_1 AP = [0.90212363 0.30555556]\n",
      "Model_2 AP = [0.9 0.1]\n"
     ]
    }
   ],
   "source": [
    "model_1_ap_2d = metrics.average_precision_score(y_true_2d, model_1_score_2d, average=None)\n",
    "model_2_ap_2d = metrics.average_precision_score(y_true_2d, model_2_score_2d, average=None)\n",
    "\n",
    "print(\"Model_1 AP = {}\".format(model_1_ap_2d))\n",
    "print(\"Model_2 AP = {}\".format(model_2_ap_2d))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`average_precision_score()` is treating each column as one binary class, we can get the same AP for the positive column if not using `average`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unique Score and PR and AUC\n",
    "\n",
    "Sometimes the results of PR and AUC can be very confusing if our `y_score` has only one unique element."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted score: [0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15]\n",
      "Predicted label: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "True label: [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n"
     ]
    }
   ],
   "source": [
    "model_1_score = [0.15 for i in range(20)]\n",
    "model_1_predict_label = [1 if i >= 0.5 else 0 for i in model_1_score]\n",
    "\n",
    "y_true = [0] + [1 for i in range(19)]\n",
    "\n",
    "print(\"Predicted score: {}\".format(model_1_score))\n",
    "print(\"Predicted label: {}\".format(model_1_predict_label))\n",
    "print(\"True label: {}\".format(y_true))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc = 0.05, AP = 0.95, AUCROC= 0.5\n"
     ]
    }
   ],
   "source": [
    "print(\"Acc = {}, AP = {}, AUCROC= {}\".format(\n",
    "    metrics.accuracy_score(y_true, model_1_predict_label),\n",
    "    metrics.average_precision_score(y_true, model_1_score),\n",
    "    metrics.roc_auc_score(y_true, model_1_score)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Even through the accuracy is extremely bad (as expected), we still get high AP and AUCROC."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precisions = [0.95 1.  ]\n",
      "recalls = [1. 0.]\n",
      "thresholds = [0.15]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAHThJREFUeJzt3Xu8VWW97/HPV0BJxaWBtJOrN7YikuYKNEspzQtboV1WkDeKpDK7nC77WFagZdbuZKd90p3sdFuJIpjbVoVZXggzNJYHJUAxIgXUfZBLmIoX5Hf+GM+CyXStMedarbHmXIvv+/WaL+e4zDF+82E5v2M8z5xjKCIwMzNry261LsDMzOqbg8LMzHI5KMzMLJeDwszMcjkozMwsl4PCzMxyOSisapKWSRpXYZ2hkp6T1KuLyiqcpMclnZyez5B0Q61rMutKDooeIH2QbUkf0P9P0vWS9u7s/UTEERExv8I6qyNi74h4tbP3nz6kX0nv86+Sfi/puM7ez64i/Z1slfTGsvmd0s6SPijpCUnPS7pN0utz1j1T0tK0z99LGlmybIqkV9Oylse4kuXDJd0j6QVJj7aEunUeB0XPcWZE7A28GWgEvly+gjLd/d/85vQ+BwD3AHNrXE+nk9S7C/axF/BeYDNwTiurtLTz/sDvgFslqR3bPwK4BjgXeAPwAnB1G+seCswCPgbsC/wcaCprh4XpAKTlMb9k2U3AYqA/cAlwi6T9q63VKuvuHxpWJiKeBG4HRgFImi/pckn3kf3PepCkBknXSnpa0pOSvl7aVSTpAkmPSPqbpOWS3pzml3bBjJHULOnZdBZzZZo/XFK0/E8u6QBJTZI2Slop6YKS/cyQNEfSj9O+lklqrPJ9biX7cBlU+qEg6QxJD5UcCY8uWTZE0q2SnpG0QdL30/yDJd2d5q2XNEvSvh1pf0kT0/6flfRnSaeVt13Je7+hrM2mSloN3C3pdkkXlW37YUnvSc8Pk/Sb1K4rJL2/naW+F/grcBlwflsrRcQrwI+AfyD7IK7W2cDPI2JBRDwHfAV4j6R+rax7KnBvRPwu/bt+CxgEnFhpJ5JGkB0cTY+ILRHxU+CP6f1ZJ3FQ9DCShgDjyY6wWpwLTAP6AU8A1wNbgUOAo4FTgI+k178PmAGcB+wDTAA2tLKr7wHfi4h9gIOBOW2UNBtYCxwAnAV8Q9I7S5ZPSOvsCzQB36/yfe6eatwAbErzjgauAz5K9qF2DdmR6R4pCH+R3v9wsg+i2S2bA65INR4ODElt0C6SxgA/Br6Q3s8JwOPt2MSJaf+nkh0lTy7Z9khgGPDLdDbwG+BGYCAwCbi6pbsmdfksqbCv89M+ZgOHSTqmjfe0BzAFWBMR6yW9LYVwW4+3pZceATzcsp2I+DPwMjCijXpU9lykg53k6BTij0n6SsnZxhHAqoj4W8m6D6f51lkiwo9u/iD7MHqO7AjxCbJT/NelZfOBy0rWfQPwUsvyNG8ycE96fgfw6Zz9nJyeLwAuBQaUrTMcCKA32Qfuq0C/kuVXANen5zOAO0uWjQS25LzPGWQfNn9N290AjCtZ/u/A18pes4LsA/g44BmgdxXt+W5gcRvvewZwQxuvuwb4bqW2K99OSZsdVLK8H/A8MCxNXw5cl55/gOwIvHzf06v8exkKbAOOKvk3/14b7bwOuBs4pp1/k3cBHyub92Tpv1fJ/MPSex0H7E529rEN+GJafhBwINmB7ZHA8pJl5wL3l23v8pa/MT865+Ezip7j3RGxb0QMi4gLI2JLybI1Jc+HAX2Ap1uOAsk+ZAam5UOAP1exv6lkR4ePSlok6YxW1jkA2Bg7H+09QXY03+K/S56/APSV1FvS2doxcHl7yTpzImJfssBbCpQeCQ8DPld6hJvezwHpv09E1rWxE0lvkDQ7dcM9C9xANgbSXtW2XVu2/zulNvsl2dkCZGE+Kz0fBowte59nk3UPVeNc4JGIeChNzwI+KKlPyTpz0t/TwIh4Z0Q82M738hzZGWmpfYC/la8YEY+SneF8H3iarO2Xk52JEhGrIuIvEbEtIv5I1l12Vnv3Yx1X+KCZ1YXSSwSvITujGNDah2ZafnDFDUb8CZisbHD8PWQDiOV92E8Br5fUryQshpIdWVba/ix2fDC2tny9pGlAs6QbI+LpVPvlEXF5+frKvrUzVFLvVt73N8ja6MiI2Cjp3VTZBVYmr+2eB/YsmW7tQ738Us43AdMlLQD6kg3et+zntxHxrg7UCFmX3VBJLSHdm6yrbjzws7wXSno72RhYW06PiHuBZcCbSl53ELAH8FhrL4qIW4Bb0rr7kh2ILGpjH8GOrqplZONupX9jbyLrlrNO4jOKXUz6QP018B1J+0jaLQ3mtgwc/hD4vKRjlDlE0rDy7Ug6R9L+EbGNrIsCsu6C0n2tAX4PXCGpbxpYnkp2xN4Z72UFWbfJv6RZ/wF8TNLYVPtekv4pDaD+gexo9Ztpfl9Jx6fX9SM7Mt0saRDZGENHXAt8SNJJqV0HSTosLXsImCSpj7IB+7Pa3sx288jOHi4j+xZSS/v+Ahgh6dy0vT6S3iLp8EobTIF5MDAGOCo9RpF9sJ5X6fURcW/s/O2j8se9adVZwJmS3p7GVC4Dbi07uyyt6xhJvZR9MWEm0JTONJB0uqQ3pOeHkXVN/SzV8xhZ205P/6b/DIwGflrpvVj1HBS7pvPI+oKXkw0E3wK8ESAi5pL18d5Idvp+G9Da999PA5ZJeo5sYHtSWXdXi8lkffBPAf9F1o9+Zye+l28D0yQNjIhm4AKys4FNwEqygVgi+13HmWQD+KvJujU+kLZxKdk3ZzaTdffc2pFCIuIPwIeA76Zt/Zbsgx6yD7eDU12XUsURb0S8lGo5uXT99GF7Clm31FNk3XffIjtiJ3XbLWtjs+cDP4uIP0bEf7c8yP4Nz1DObx3aIyKWkX3ddRbZOEc/4MKW5elbXV8qecn3yA44VpC10QUly04Clkh6niw8byU7C2wxiewr4ZuAbwJnRcQznfE+LKMI37jIzMza5jMKMzPL5aAwM7NcDgozM8vloDAzs1zd7ncUAwYMiOHDh9e6DDOzbuXBBx9cHxEdulhitwuK4cOH09zcXOsyzMy6FUlPdPS17noyM7NcDgozM8vloDAzs1wOCjMzy+WgMDOzXA4KMzPLVVhQSLpO0jpJS9tYLkn/puw+ykuU7stsZmb1pcgziuvJLkXdltOBQ9NjGtltLM3MrM4UFhQRsQDYmLPKRODHkbkf2FfSGytt9+WXO6tCMzOrRi3HKAax872c17LzvZS3kzRNUrOk5qef3tQlxZmZWaZbDGZHxMyIaIyIxoaG/WpdjpnZLqWWQfEkMKRkenCaZ2ZmdaSWQdEEnJe+/XQssDkinq5hPWZm1orCrh4r6SZgHDBA0lpgOtAHICJ+QHaT9PHASuAFspvSm5lZnSksKCJicoXlAXyiqP2bmVnn6BaD2WZmVjsOCjMzy+WgMDOzXN3uVqjbtsFjj9W6CjOzXUe3CwqABQtqXYGZWXfTb6+OvrLbBUXv3nD00bWuwsysu+nVq6Ov9BiFmZnlclCYmVkuB4WZmeVyUJiZWS4HhZmZ5XJQmJlZLgeFmZnlclCYmVkuB4WZmeVyUJiZWS4HhZmZ5XJQmJlZLgeFmZnlclCYmVkuB4WZmeVyUJiZWS4HhZmZ5XJQmJlZLgeFmZnlclCYmVkuB4WZmeVyUJiZWS4HhZmZ5XJQmJlZLgeFmZnlclCYmVmuQoNC0mmSVkhaKeniVpYPlXSPpMWSlkgaX2Q9ZmbWfoUFhaRewFXA6cBIYLKkkWWrfRmYExFHA5OAq4uqx8zMOqbIM4oxwMqIWBURLwOzgYll6wSwT3reADxVYD1mZtYBvQvc9iBgTcn0WmBs2TozgF9L+iSwF3ByaxuSNA2YBjBw4NBOL9TMzNpW68HsycD1ETEYGA/8RNJraoqImRHRGBGNDQ37d3mRZma7siKD4klgSMn04DSv1FRgDkBELAT6AgMKrMnMzNqpyKBYBBwq6UBJu5MNVjeVrbMaOAlA0uFkQfFMgTWZmVk7FRYUEbEVuAi4A3iE7NtNyyRdJmlCWu1zwAWSHgZuAqZERBRVk5mZtV+Rg9lExDxgXtm8r5Y8Xw4cX2QNZmb296n1YLaZmdU5B4WZmeVyUJiZWS4HhZmZ5XJQmJlZLgeFmZnlclCYmVkuB4WZmeVyUJiZWS4HhZmZ5XJQmJlZLgeFmZnlclCYmVkuB4WZmeVyUJiZWS4HhZmZ5XJQmJlZLgeFmZnlclCYmVkuB4WZmeVyUJiZWS4HhZmZ5XJQmJlZLgeFmZnlclCYmVkuB4WZmeVyUJiZWS4HhZmZ5XJQmJlZLgeFmZnlclCYmVmu3tWuKGkQMKz0NRGxoIiizMysflQVFJK+BXwAWA68mmYHkBsUkk4Dvgf0An4YEd9sZZ33AzPS9h6OiA9WW7yZmRWv2jOKdwP/GBEvVbthSb2Aq4B3AWuBRZKaImJ5yTqHAl8Ejo+ITZIGVl+6mZl1hWrHKFYBfdq57THAyohYFREvA7OBiWXrXABcFRGbACJiXTv3YWZmBav2jOIF4CFJdwHbzyoi4lM5rxkErCmZXguMLVtnBICk+8i6p2ZExK+qrMnMzLpAtUHRlB5F7P9QYBwwGFgg6ciI+GvpSpKmAdMABg4cWkAZZmbWlqqCIiJ+JGl30hkAsCIiXqnwsieBISXTg9O8UmuBB9K2/iLpMbLgWFS2/5nATIARIxqjmprNzKxzVDVGIWkc8CeywemrgccknVDhZYuAQyUdmEJmEq89K7mN7GwCSQPIgmhVtcWbmVnxqu16+g5wSkSsAJA0ArgJOKatF0TEVkkXAXeQjT9cFxHLJF0GNEdEU1p2iqSWr91+ISI2dPztmJlZZ6s2KPq0hARARDwmqeK3oCJiHjCvbN5XS54H8Nn0MDOzOlRtUDRL+iFwQ5o+G2gupiQzM6sn1QbFx4FPAC1fh72XbKzCzMx6uGq/9fQScGV6mJnZLiQ3KCTNiYj3S/oj2bWYdhIRowurzMzM6kKlM4pPp/+eUXQhZmZWn3J/RxERT6en64E1EfEEsAfwJuCpgmszM7M6UO1FARcAfdM9KX4NnAtcX1RRZmZWP6oNCkXEC8B7gKsj4n3AEcWVZWZm9aLqoJB0HNnvJ36Z5vUqpiQzM6sn1QbFZ8huMPRf6TIcBwH3FFeWmZnVi2p/R/Fb4Lcl06vY8eM7MzPrwSr9juJ/R8RnJP2c1n9HMaGwyszMrC5UOqP4Sfrv/yq6EDMzq0+5QRERD6anzcCWiNgGIKkX2e8pzMysh6t2MPsuYM+S6dcBd3Z+OWZmVm+qDYq+EfFcy0R6vmfO+mZm1kNUGxTPS3pzy4SkY4AtxZRkZmb1pNr7UXwGmCvpKUDAPwAfKKwqMzOrG9X+jmKRpMOAf0yzVkTEK8WVZWZm9aKqridJewL/E/h0RCwFhkvypcfNzHYB1Y5R/CfwMnBcmn4S+HohFZmZWV2pNigOjoh/BV4BSFeSVWFVmZlZ3ag2KF6W9DrSZTwkHQy8VFhVZmZWN6r91tN04FfAEEmzgOOBKUUVZWZm9aNiUEgS8CjZTYuOJety+nRErC+4NjMzqwMVgyIiQtK8iDiSHTctMjOzXUS1YxT/V9JbCq3EzMzqUrVjFGOBcyQ9DjxP1v0UETG6qMLMzKw+VBsUpxZahZmZ1a1Kd7jrC3wMOAT4I3BtRGztisLMzKw+VBqj+BHQSBYSpwPfKbwiMzOrK5W6nkambzsh6VrgD8WXZGZm9aTSGcX2K8S6y8nMbNdUKSjeJOnZ9PgbMLrluaRnK21c0mmSVkhaKeninPXeKykkNbb3DZiZWbFyu54ioldHNyypF3AV8C5gLbBIUlNELC9brx/waeCBju7LzMyKU+0P7jpiDLAyIlZFxMvAbGBiK+t9DfgW8GKBtZiZWQcVGRSDgDUl02vTvO3SfbiHRETupUEkTZPULKl58+ZnOr9SMzNrU5FBkUvSbsCVwOcqrRsRMyOiMSIaGxr2L744MzPbrsigeBIYUjI9OM1r0Q8YBcxPlwY5FmjygLaZWX0pMigWAYdKOlDS7sAkoKllYURsjogBETE8IoYD9wMTIqK5wJrMzKydCguK9LuLi4A7gEeAORGxTNJlkiYUtV8zM+tc1V4UsEMiYh4wr2zeV9tYd1yRtZiZWcfUbDDbzMy6BweFmZnlclCYmVkuB4WZmeVyUJiZWS4HhZmZ5XJQmJlZLgeFmZnlclCYmVkuB4WZmeVyUJiZWS4HhZmZ5XJQmJlZLgeFmZnlclCYmVkuB4WZmeVyUJiZWS4HhZmZ5XJQmJlZLgeFmZnlclCYmVkuB4WZmeVyUJiZWS4HhZmZ5XJQmJlZLgeFmZnlclCYmVkuB4WZmeVyUJiZWS4HhZmZ5XJQmJlZLgeFmZnlKjQoJJ0maYWklZIubmX5ZyUtl7RE0l2ShhVZj5mZtV9hQSGpF3AVcDowEpgsaWTZaouBxogYDdwC/GtR9ZiZWccUeUYxBlgZEasi4mVgNjCxdIWIuCciXkiT9wODC6zHzMw6oMigGASsKZlem+a1ZSpwe2sLJE2T1CypefPmZzqxRDMzq6QuBrMlnQM0At9ubXlEzIyIxohobGjYv2uLMzPbxfUucNtPAkNKpgeneTuRdDJwCXBiRLxUYD1mZtYBRZ5RLAIOlXSgpN2BSUBT6QqSjgauASZExLoCazEzsw4qLCgiYitwEXAH8AgwJyKWSbpM0oS02reBvYG5kh6S1NTG5szMrEaK7HoiIuYB88rmfbXk+clF7t/MzP5+dTGYbWZm9ctBYWZmuRwUZmaWy0FhZma5HBRmZpbLQWFmZrkcFGZmlstBYWZmuRwUZmaWy0FhZma5HBRmZpbLQWFmZrkKvShg13mF3XZbi/QiUq1rqW8RENGXbdsGA31qXY6ZdQM9Iih2220tAwf2o6FhOHJS5IoINm/ewLp1a9m27cBal2Nm3UCP6HqSXqShob9DogqSUlu9WOtSzKyb6CFBgUOiHSS5i87MqtYjgsLMzIrTI8Yoyv3hD7B5c+dtr6EBxozpvO2ZmXUnPfKMYvNmGDCg8x7VhM7ee4upU8/ZPr1161aGDdufs846o121jxw5nPXr1/9d63z84x9m+PCBvOUto9q1bzOz1vTIoKiFvfbai+XLl7JlyxYA7r77NxxwwKCa1HL22VO47bZf1WTfZtbzOCg60amnjudXv/olAHPn3sT73jd5+7KNGzcyadK7GTt2NO94x7EsXboEgA0bNjBhwik0Nh7BJz7xESJi+2tmz76BE08cw3HHHcUnP/lRXn311arqeNvbTmC//V7fie/MzHZlDopOdNZZk7jlltm8+OKLLF26hMbGsduXXX75dEaPPpoHHljC9Onf4IILzgPgiisu5bjj3kZz8zLOPPOfWbNmNQCPPvoIP/3pzdx5530sXPgQvXr14uabZ9XkfZnZrq1HDmbXyqhRo1m9+nHmzr2JU08dv9OyhQt/x6xZPwVg3Lh3snHjBp599lnuu28BN954KwCnnfZP7LfffgDMn38Xixc/yAknvAWAF1/cwv77D+zCd2NmlnFQdLLx4ydwySWfZ968+WzcuKHD24kIzj77fC699IpOrM7MrP16ZNdTQwOsX995j4aG6vd93nkf5uKLpzNq1JE7zX/rW9++vetowYL59O8/gH322Yfjjz+BOXNuBODXv76dTZs2ATBu3EncdtstrFu3DsjGOFavfqITWsfMrH165BlFLX/zMGjQYC688FOvmf+lL83gwgs/zNixo9lzzz2ZOfNHAHzxi9P50Icm09h4BMce+1aGDBkKwOGHj+QrX/k6EyeewrZt2+jTpw9XXnkVQ4cOq1jDlCmTuffe+WzYsJ4RIwZzySWXcv75Uzv3jZrZLkOl37LpDkaMaIybbmreaV7v3o9wyCGH16ii7mnlykfYutVtZraraGzc708Rm0Z05LU9suvJzMw6T4/setoVbNiwgTPOOOk183/xi7vo379/DSoys56qRwRFdjOe2KWuINu/f38WLnyoQ6+NCLpZj6OZ1VCP6HqK6MvmzRvobuMttdBy46KIvrUuxcy6iR5xRrFt22DWrVvLM8884/ssVLDzrVDNzCrrEUEBfXxbTzOzghTa9STpNEkrJK2UdHEry/eQdHNa/oCk4UXWY2Zm7VdYUEjqBVwFnA6MBCZLGlm22lRgU0QcAnwX+FZR9ZiZWccUeUYxBlgZEasi4mVgNjCxbJ2JwI/S81uAk7QrfXXJzKwbKHKMYhCwpmR6LTC2rXUiYqukzUB/YKfbt0maBkxLU680Nu73eCEVdzsvNcAenXjT1+7MbbGD22IHt8UOz3b4GyzdYjA7ImYCMwEkNUdsaqxxSXUha4sX3Ba4LUq5LXZwW+wgqbnyWq0rsuvpSWBIyfTgNK/VdST1BhqAjl+b28zMOl2RQbEIOFTSgZJ2ByYBTWXrNAHnp+dnAXeHfzVnZlZXCut6SmMOFwF3AL2A6yJimaTLgOaIaAKuBX4iaSWwkSxMKplZVM3dkNtiB7fFDm6LHdwWO3S4LbrdZcbNzKxr9YhrPZmZWXEcFGZmlqtug8KX/9ihirb4rKTlkpZIuktS5fuldlOV2qJkvfdKCkk99quR1bSFpPenv41lkm7s6hq7ShX/jwyVdI+kxen/k/G1qLNokq6TtE7S0jaWS9K/pXZaIunNVW04uzdBfT3IBr//DBwE7A48DIwsW+dC4Afp+STg5lrXXcO2eAewZ3r+8V25LdJ6/YAFwP1AY63rruHfxaHAYmC/ND2w1nXXsC1mAh9Pz0cCj9e67oLa4gTgzcDSNpaPB24HBBwLPFDNduv1jMKX/9ihYltExD0R8UKavJ/sNys9UTV/FwBfI7tu2ItdWVwXq6YtLgCuiohNABGxrotr7CrVtEUA+6TnDcBTXVhfl4mIBWTfIG3LRODHkbkf2FfSGyttt16DorXLfwxqa52I2Aq0XP6jp6mmLUpNJTti6IkqtkU6lR4SEb/sysJqoJq/ixHACEn3Sbpf0mldVl3XqqYtZgDnSFoLzAM+2TWl1Z32fp4A3eQSHlYdSecAjcCJta6lFiTtBlwJTKlxKfWiN1n30ziys8wFko6MiL/WtKramAxcHxHfkXQc2e+3RkXEtloX1h3U6xmFL/+xQzVtgaSTgUuACRHxUhfV1tUqtUU/YBQwX9LjZH2wTT10QLuav4u1QFNEvBIRfwEeIwuOnqaatpgKzAGIiIVAX2BAl1RXX6r6PClXr0Hhy3/sULEtJB0NXEMWEj21HxoqtEVEbI6IARExPCKGk43XTIiIDl8MrY5V8//IbWRnE0gaQNYVtaori+wi1bTFauAkAEmHkwXFM11aZX1oAs5L3346FtgcEU9XelFddj1FcZf/6HaqbItvA3sDc9N4/uqImFCzogtSZVvsEqpsizuAUyQtB14FvhARPe6su8q2+BzwH5L+B9nA9pSeeGAp6Sayg4MBaTxmOtAHICJ+QDY+Mx5YCbwAfKiq7fbAtjIzs05Ur11PZmZWJxwUZmaWy0FhZma5HBRmZpbLQWFmZrkcFGZlJL0q6SFJSyX9XNK+nbz9KZK+n57PkPT5zty+WWdzUJi91paIOCoiRpH9RucTtS7IrJYcFGb5FlJy0TRJX5C0KF3L/9KS+eeleQ9L+kmad2a6V8piSXdKekMN6jf7u9XlL7PN6oGkXmSXfbg2TZ9Cdq2kMWTX82+SdALZNca+DLw1ItZLen3axO+AYyMiJH0E+BeyXwibdSsOCrPXep2kh8jOJB4BfpPmn5Iei9P03mTB8SZgbkSsB4iIlvsBDAZuTtf73x34S9eUb9a53PVk9lpbIuIoYBjZmUPLGIWAK9L4xVERcUhEXJuznf8DfD8ijgQ+SnYhOrNux0Fh1oZ018BPAZ9Ll7K/A/iwpL0BJA2SNBC4G3ifpP5pfkvXUwM7LuF8PmbdlLuezHJExGJJS4DJEfGTdInqhekqvc8B56QrlV4O/FbSq2RdU1PI7qo2V9ImsjA5sBbvwezv5avHmplZLnc9mZlZLgeFmZnlclCYmVkuB4WZmeVyUJiZWS4HhZmZ5XJQmJlZrv8PBWJRW2tp2GsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1067afe10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "precisions, recalls, thresholds_pr = metrics.precision_recall_curve(y_true, model_1_score)\n",
    "print(\"precisions = {}\".format(precisions))\n",
    "print(\"recalls = {}\".format(recalls))\n",
    "print(\"thresholds = {}\".format(thresholds_pr))\n",
    "\n",
    "# In matplotlib < 1.5, plt.fill_between does not have a 'step' argument\n",
    "step_kwargs = ({'step': 'post'}\n",
    "               if 'step' in signature(plt.fill_between).parameters\n",
    "               else {})\n",
    "\n",
    "plt.step(recalls, precisions, color='blue', alpha=0.2, where='post')\n",
    "plt.fill_between(recalls, precisions, alpha=0.2, color='blue', label=\"Model_1\", **step_kwargs)\n",
    "\n",
    "\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.legend()\n",
    "plt.title('Precision-Recall curve: AP={:.3f}'.format(metrics.average_precision_score(y_true, model_1_score)))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Because there is only one unique prediction score, we only get one threshold which is exactly that unique score. At this threshold, all scores ($0.15$) are translated to the positive label ($1$). Since there is only one negative label in `y_true`, the precision at this threshold is $\\frac{\\text{TP}}{\\text{TP} + \\text{FP}} = \\frac{19}{19+1} = 0.95$\n",
    "\n",
    "Therefore, the AP score is simply given by $\\left(1 - 0\\right) \\times \\text{Precision} = 0.95$.\n",
    "\n",
    "This tells us when `y_score` has only one unique value, PR curve and AP only depend on the number of positives in `y_true`. Given this condition,\n",
    "\n",
    "$$\\text{Precision} = \\frac{\\text{TP}}{\\text{TP} + \\text{FP}} = \\frac{N_p}{N_p + \\left(N - N_p\\right)} = \\frac{N_p}{N}$$\n",
    "\n",
    "Interestingly, \n",
    "\n",
    "$$\\text{ACC} = \\frac{\\text{TP} + \\text{TN}}{\\text{N}}$$\n",
    "\n",
    "If the value of `y_score` $\\geq 0.5$, then all predictions in ACC becomes $1$.\n",
    "\n",
    "$$\\text{ACC} = \\frac{N_p}{N} = \\text{Precision}$$\n",
    "\n",
    "If the value of `y_score` $< 0.5$, then all predictions in ACC becomes $0$.\n",
    "\n",
    "$$\\text{ACC} = \\frac{N_n}{N} = 1 - \\frac{N_p}{N} = 1 - \\text{Precision}$$\n",
    "\n",
    "$N$ is the size of `y_true`, $N_p$ is the number of positives in `y_true`, and $N_n$ is number of negatives in `y_true`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FP Rates = [0.95 1.  ]\n",
      "TP Rates = [1. 0.]\n",
      "thresholds = [0.15]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VPXZ///XZQgJhIRAFpasM2FPQNCAaN0qLogKotiqtdZqa+23223v9ne3Im5tXbrYW+/aWqtWbV3aWlkUFDcQN2RRCknY9yBrgBDIQpbP7485yBhDGCCTSSbv5+MxD87yOedc5zCZa875nHONOecQERE5kpMiHYCIiLRtShQiItIsJQoREWmWEoWIiDRLiUJERJqlRCEiIs1SohARkWYpUQhmtsHMqsxsv5ltM7OnzKxbozZnmNnbZlZhZuVm9rKZDWnUJsnM/tfMNnnrWuuNp7buHrU8M7vBzJyZfbWJ6e810X6DmZ0fND7KzGaZ2V4z221mC8zsm968c82swTtmFWa28tC8oOXNzH5qZqu9/6tNZnafmcU1anfE7Rzj/o4xsxVmVmlmc8wsp5m2we+f/Wb2eqP5t3rvq31m9mRwzGaW662/0tve+V/cgkSaEoUccplzrhswHBgB/PzQDDM7HXgdmA70BXzAf4D3zczvtekMvAXkA2OBJOB0oAwYFa6gzaxTuNbdyDeA3cD1x7qgd/zeBt4B+gEpwHeBi4Oafeod/yTgVuAvZjYwaP7DwM3e9hO9ZccA/zzG7YQSbyrwEjAF6AksAv5xlMUuc851814XBq3rIuBnXqw5gB+4O2i554FPvFgnAy+aWdqxxCutwDmnVwd/ARuA84PGfw3MDBp/F/hjE8u9CjzjDX8L2A50O4bt5gNvEPgA3g7c5k1/CvhlULtzgdJG8f4PsBSo8YZfbLTuh4CHveHuwBPAVmAL8Esg5hjizAEagCuBOqB30LwbgPeaO6bAe8Ajzaz/c/vnTdsBXOUN9wfqgVGN2mR5+39eKNs5hv29GfggaDwBqAIGhfL+aTTvOeDeoPExwDZveIAXf2Kj99otkf6b0OvzL51RyOeYWSaBb6BrvPGuwBnAv5po/k/gAm/4fOA159z+ELeTCLwJvEbgLKUfgTOSUF0DXAIkAy8A47x1YmYxwFcIfEhBIPHUedsYAVxIILFhZtneZZrsZrZ1PbDIOfdvYDnwtVCD9I7f6cCLIbY/yczGA6l4/wcEPlxLnXMLgts65zYD84ELQtlO0L4e6XWt1zSfwBnjoe0cANZ604/kWTPbaWavm9nJQdM/ty5vuJeZpXjz1jnnKhrNb247EgFKFHLINDOrADYT+DZ7pze9J4H3ydYmltlK4AMNApcOmmpzJJcS+Gb5O+dctXOuwjn30TEs/7BzbrNzrso5txH4GJjozTsPqHTOzTezXsA44L+ccwecczuA3wNXAzjnNjnnkp1zm5rZ1vUcTjrPcWyXn3pw5OMXrK+Z7SXwzX0q8GPn3CfevNRmlj/0f3DU7QTt65Feh/axG1DeaPFyApe8mvI1IJfAmdccYLaZJR9hXYeGE49jOxIhShRyyOXOuUQCl0EGcTgB7CFw2aVPE8v0AXZ5w2VHaHMkWQS+pR6vzY3GnyNwlgFwLYc/2HOAWGDroW/OwJ+B9FA2YmZfItAn80LQdoaa2XBvvM5bf2OxQC3NH79gnzrnkgn0UTxMINkdsquZ5Q/9H4S6nVDs9+IIlgRUNNEW59z7XsKudM7dB+wFzjrCug4NVxzrdiRylCjkc5xz7xC4VPNbb/wA8CFwVRPNv8Lhy0VvAheZWUKIm9pMoGOzKQeArkHjvZsKtdH4v4BzvUtnEzmcKDYTuA6eGvTNOck5F+rljW8ABiwxs23AR0HTATYB2WZmhxbwLgOlAxudc5UEjt+VoWzMOXeoz2WomV3uTX4byDKzz90UYGZZwGjgrVC241162t/M69AltWLg5KDlEoA8b3pIu0HgmH1hXd7wdudcmTfPf+iSYdD8ULcjrSXSnSR6Rf7FFzuz0wh8WJ/sjZ/pjf+QwGWBHgQ6hPcC/b02ccBCAn0Ogwh8CUkBbgPGNbHNRAKXSf7LWzYROM2b921gBYHLXr0JXIdv3Jn9hc5TAp3rbwCfNJo+nUDndpIXVx5wTgjHJd7bx5u8OA69vkeg872TF/t6AneJxRPo+H2YwIe2ees5g8C3558CKd60k4EXvOFz+WJn9veBj4PG/wisJpAYYghcx1/A5286aHY7x/B+SCNwCehKb58eAOYfoW028CWgs9f2p8DOoO2PBbYBQwj0J70N3B+0/HwCX0riCST4vUBapP8m9Gr0/xzpAPSK/KupD17gT8C/g8bPBOZ6H0T7gJlAQaNlugP/S+Bb/H4Cl5YePPSh0cR2CwickezxPkx+5k2PJ3A75j4CdzbdSmiJ4usEvs3+tIm4/gSUeh+AnwBXe/OyvVizm1jf1QSSWWyj6V0IXGq71BsfAswmcAloO4EO5axGy4wikMjKCdzl9RFwvTfvXL6YKLp667vMGz+JwJnGGgL9GJsJ3J0WH+p2jvE9cT6BZF3l/b/nBs17FHjUG873/o8OeMfkLaCw0bp+7B2XfcBfgbigebne+quAlU39v+oV+dehbzwiIiJNUh+FiIg0S4lCRESapUQhIiLNUqIQEZFmtVZBtRaTmprqcnNzIx2GiEi7snjx4l3OueMquNjuEkVubi6LFi2KdBgiIu2KmW083mV16UlERJqlRCEiIs1SohARkWa1uz6KptTW1lJaWkp1dXWkQ2kX4uPjyczMJDa2qaKnIiKfFxWJorS0lMTERHJzcwkq4ilNcM5RVlZGaWkpPp8v0uGISDsQFZeeqqurSUlJUZIIgZmRkpKisy8RCVnYEoWZPWlmO8ys6AjzzcweNrM1ZrbUzE45we2dyOIdio6ViByLcJ5RPEWgFv2RXEzgR+P7E/gx9z+FMRYRkQ5rf03dCS0ftj4K59w8M8ttpskE4BkXqHM+38ySzayPc+5Yfne5SfPnw969J7qWw5KTYfTollufiEhrcM4xu3gbd80oAY7/UkIk+ygy+PzvHpd6077AzG42s0Vmtmjnzp1HXfHevZCW1nKvUJKOmXHdddd9Nl5XV0daWhqXXnrp0RcOkpuby65du06ozY033kh6ejoFBQXHtG0RiR6bd1dy09OLuOXvHxNnsUC3Lse7rnbRme2ce8w5V+icK0xLO65SJWGXkJBAUVERVVVVALzxxhtkZDSZ98Luhhtu4LXXXovItkUksmrrG/jT3LVc8Pt3+GBNGVf6B/M/w88E7Lg/7yOZKLYAWUHjmd60dmvcuHHMnDkTgOeff55rrrnms3m7d+/m8ssvZ9iwYYwePZqlS5cCUFZWxoUXXkh+fj7f+ta3CP7Fwb///e+MGjWK4cOH853vfIf6+vqQ4jj77LPp2bNnC+6ZiLQHCzfs5pKH3+WB11YwpGcaU049h68O99Ozx4l91EcyUcwArvfufhoNlLdE/0QkXX311bzwwgtUV1ezdOlSTjvttM/m3XnnnYwYMYKlS5dy7733cv311wNw9913c+aZZ1JcXMzEiRPZtGkTAMuXL+cf//gH77//PkuWLCEmJoZnn302IvslIm3bngMH+Z8Xl3LVox+yp6KeW/IL+fHIQgZldyEm5sTXH7bObDN7nsCPxqeaWSlwJxAL4Jx7FJgFjCPwY/GVwDfDFUtrGTZsGBs2bOD5559n3Lhxn5v33nvv8e9//xuA8847j7KyMvbt28e8efN46aWXALjkkkvo0aMHAG+99RaLFy9m5MiRAFRVVZGent6KeyMibZ1zjhcXl3LvrOXsq6rjomw/4/P60yul0wl0XX9ROO96uuYo8x3wvXBtP1LGjx/PT37yE+bOnUtZWdlxr8c5xze+8Q3uu+++FoxORKLFmh0V3Da1iAXrdzOgZw++N7iAgqwkwlGZJypKeDSWnAwh3Bx1TOsL1Y033khycjJDhw5l7ty5n00/66yzePbZZ5kyZQpz584lNTWVpKQkzj77bJ577jluv/12Xn31Vfbs2QPAmDFjmDBhArfeeivp6ens3r2biooKcnJyWm7HRKTdqa6t5w9vr+HP89YSF9OJr/Ufyvl5WSQlhu9B2qhMFJF85iEzM5Mf/vCHX5h+1113ceONNzJs2DC6du3K008/DQT6Lq655hry8/M544wzyM7OBmDIkCH88pe/5MILL6ShoYHY2FgeeeSRkBLFNddcw9y5c9m1axeZmZncfffd3HTTTS27oyLS6uau3MEd04vZtLuSM/pkMNE/mNzecZwU5t5mC77Lpj0oLCx0jX/hbvny5QwePDhCEbVPOmYi7cf2fdXc80oJM5duJSMxgUm5BYzKTSU+PrTla2vh9NO7r3SufNDxbD8qzyhERKJBfYPjbx9u4Levr6KmroHLcgdwSZ6ftJ4tcCvTMVCiaKfKysoYM2bMF6a/9dZbpKSkRCAiEWlJy0rLuW3qMpZtKacgNZWr/AUMykigUwQ+taMmUTjnOlRV1JSUFJYsWXJcy7a3y40iHcm+6loefH0Vz3y4ge7xcdw4aATn+PrQrVvkPt+iIlHEx8dTVlam36QIwaEfLooP9eKmiLQK5xyzlm3j7peL2VlRw7mZOUzwDSSzV2yLPhNxPKIiUWRmZlJaWkooBQPl8E+hikjbsKmskinTi3hn1U58yUncMKKQEdnJdO4c6cgCoiJRxMbG6mc9RaTdOVjXwF/eXcfDb63mJDMm5Q1hrD+HHsltq15rVCQKEZH25qN1ZUyeVsSaHfsp7NWbK/355PWJb5HaTC1NiUJEpBXtPnCQe2ct58XFpaQndOG7+YWc6e8V8jMRkaBEISLSChoaDhfwq6ipY2xOHpf5+9MrJSbindVHo0QhIhJmq7ZXMHnqMhZu2MPAnj34Qf5QCrISI/JMxPFoJ2GKiLQ/VQfrefjt1fxl3jq6xHbi6wOGcZ4/M6wF/MJBiUJEJAzeXrGdO6YXU7qnii/1zeQK32Cye3cOewG/cFCiEBFpQVvLq7h7RgmvFW8jM7Eb/zV0NKN8KcTFRTqy46dEISLSAurqG3j6w408+PpKausdl/sGcrHfT2rPdngK0YgShYjICVqyeS+Tpy6j+NN9DEtLY5KvgIEZXdtNZ/XRRMluiIi0vn3Vtfx29kr+Nn8jPePj+NbgUzjb15uEhPbVWX00ShQiIsfIOcfLS7fyi1dKKNtfw5czcpngH0BGeuQL+IWDEoWIyDHYsOsAU6YX8e7qXfiTu3Pj8EJG5CQTGxvpyMJHiUJEJAQ1dfX8+Z11/GHOGjrZSXy1Xz4X5eXQPSkKTyEaUaIQETmKD9bu4vZpRazbeYCRvftwpW8I/jZawC8clChERI5g1/4a7p25nJc+2UKvhK58r2AkZ/jS23QBv3BQohARaaShwfGPRZu5/9UVHKip4+Kcflzm70d6OyjgFw5KFCIiQVZs28fkqUUs3riHwSk9mZRf0K4K+IVDB951EZHDKg/W8dCbq3n8vfV0i43l+gEnc54/g8R2VsAvHJQoRKTDe7NkO3fMKObTvVWclZHFRP8gstLbZwG/cFCiEJEO69O9Vdw1o5jXS7aTldSNH598OoU5Pdt1Ab9wUKIQkQ6nrr6Bpz7YwINvrKK+3jHRP4iL/T5SeugUoilKFCLSoXy8aQ+TpxaxfOs+Tk5L5yp/Pv37Rk8Bv3DQoRGRDqG8spYHZq/g+QWb6Nklnm8POYWzcqOvgF84hDVRmNlY4CEgBnjcOXd/o/nZwNNAstfmZ865WeGMSUQ6Fucc05d8yi9mlrDnwEHGZPqY4B9An7ROHfKZiOMRtkRhZjHAI8AFQCmw0MxmOOdKgprdDvzTOfcnMxsCzAJywxWTiHQs63buZ8r0It5fU0Zej+7cPGIUJ2d3j+oCfuEQzjOKUcAa59w6ADN7AZgABCcKByR5w92BT8MYj4h0ENW19Tz6zloembOWzjEncXX/Ai70Z3eIAn7hEM5EkQFsDhovBU5r1OYu4HUz+wGQAJzf1IrM7GbgZoDs7OwWD1REosd7q3cxZXoR63cd4LTefbnSPxhfn3g9E3ECIt2ZfQ3wlHPud2Z2OvA3MytwzjUEN3LOPQY8BlBYWOgiEKeItHE7K2r45cwSpi/5lN7duvL9glGc7kvrcAX8wiGciWILkBU0nulNC3YTMBbAOfehmcUDqcCOMMYlIlGkocHx3IJNPPDaCqoO1nNJTj8u8fejV2oHqQHeCsKZKBYC/c3MRyBBXA1c26jNJmAM8JSZDQbigZ1hjElEokjJp/u4beoylmzey5CUFCYVFJCf1U3PRLSwsB1O51ydmX0fmE3g1tcnnXPFZnYPsMg5NwP4b+AvZnYrgY7tG5xzurQkIs06UFPH799YxV8/2EC3zrF8Y2CggF+3buqsDoew5l3vmYhZjabdETRcAnwpnDGISHSZXbyNu2YUs7W8mnMysrncP5BMFfALK52giUi7ULqnkrtmlPDm8u1kd0/kv08ewakq4NcqlChEpE2rrW/gyffW879vrsY5uMI/iLEq4NeqlChEpM1avHE3k6cWsWJbBSPSe3GVP59+fbsQoxuaWpUShYi0OXsrD/LAayt4fsFmUrvGc/OQUznb35suXSIdWcekRCEibYZzjqmfbOFXM5ezt7KWC7N9XOZTAb9IU6IQkTZhzY793D5tGfPX7aZ/j2S+M3Aow7KTVMCvDVCiEJGIqq6t55E5a3j0nbXExcRwbf8CLsjLJilRpxBthRKFiETMvFU7mTK9iI1llYzu05cr/UPI7R2nZyLaGCUKEWl1O/ZVc88rJbyydCt9uiXww6GncVpuqgr4tVFKFCLSauobHM99tJEHXlvJwboGLssdwCV+P2kput+1LVOiEJFWUbSlnMlTl/Gf0nIKUlO5ylfAoMwEFfBrB/RfJCJhtb+mjgdfX8VTH6yne1wc3xw0nHN9fVXArx1RohCRsHDO8VrRNu5+uYTt+6o5JzObCbmDyOodq2ci2hklChFpcZt3V3LH9CLmrNxJTvckfjL8FE7J6UHnzpGOTI6HEoWItJiDdQ08/t46Hn5rNThjUt5gxvpz6ZGs+13bMyUKEWkRC9bv5vZpy1i1fT+n9urFFbn59M9QAb9ooEQhIidkz4GD3Pfqcv65qJS0rl24Jb+QM329VMAviihRiMhxcc7x4uJS7p21nH3VdVyU7Wd8Xn96paiAX7RRohCRY7ZmRwW3TS1iwfrdDOjZg+8NLmBodpKeiYhS+m8VkZBVHaznD3NW89i8dcTFdOJr/Ydyfl6WCvhFOSUKEQnJ3JU7mDKtiM17qvhS30wm+gaRowJ+HYIShYg0a/u+au55uYSZy7aSkZjAj4aOZlRuigr4dSBKFCLSpPoGx98+3MBvX19FTV0D43MHMC7PT1pP3e/a0ShRiMgXLC3dy+SpRSzbUs7QtFSuzC1gsAr4dVj6bxeRz+yrruV3s1fyzPyNJMfHceOgEZzr70NCgjqrOzIlChHBOcesZdu4++VidlbU8OXMHMb7B5KZrgJ+okQh0uFtKqtkyvQi3lm1E1/3JG4YXsiInGQV8JPPKFGIdFAH6xr4y7uBAn4xdhJX5Q3hIn+OCvjJFyhRiHRA89eVcfu0Itbs2E9hr95c6c8nr0+8CvhJk5QoRDqQsv013DtrBf/+uJT0hC58N38kZ/rT9UyENEuJQqQDaGhw/GvxZu6btYKKmjouzsnjUn9/eqXEqLNajkqJQiTKrdxWwe3TlrFwwx4G9uzBD/KHUpCVqGciJGRH7bUysx+ZWZIFPGFmH5vZhaGs3MzGmtlKM1tjZj87QpuvmFmJmRWb2XPHugMi0rSqg/Xc/+oKLnn4XVZu3c/XBw7jZ6NOZ7hPSUKOTShvlxudcw+Z2UVAD+DrwN+A15tbyMxigEeAC4BSYKGZzXDOlQS16Q/8HPiSc26PmaUf536ISJC3V2znjunFlO6p4sy+mUz0Dya7V2cV8JPjEkqiOHQFcxzwN+dcsVlIVzVHAWucc+sAzOwFYAJQEtTm28Ajzrk9AM65HSFHLiJfsLW8irtnlPBa8TYyE7tx67DRjMxNIS4u0pFJexZKolhsZq8DPuDnZpYINISwXAawOWi8FDitUZsBAGb2PhAD3OWcey2EdYtIkLr6Bp7+cCMPvr6S2nrH5b6BXOz3k9pTpxBy4kJJFDcBw4F1zrlKM0sBvtmC2+8PnAtkAvPMbKhzbm9wIzO7GbgZIDs7u4U2LRIdlmzey+Spyyj+dB/D0tKY5CtgYEZX9UNIiwnlreSAIcClwD1AAhDKXddbgKyg8UxvWrBS4CPnXC2w3sxWEUgcCz8XgHOPAY8BFBYWuhC2LRL1yqtq+e3slfz9o430iI/jW4NP4WxfbxXwkxYXSqL4I4FLTecRSBQVwL+BkUdZbiHQ38x8BBLE1cC1jdpMA64B/mpmqQQuRa0LOXqRDsg5x8tLt/KLV0oo21/DeZm5jPcNIEMF/CRMQkkUpznnTjGzTwC8u5OOWi7MOVdnZt8HZhPof3jS6wi/B1jknJvhzbvQzEqAeuCnzrmy494bkSi3YdcBpkwv4t3Vu/And+em4SMZntOd2NhIRybRLJREUevd6uoAzCyN0Dqzcc7NAmY1mnZH0LADfuy9ROQIaurq+fM76/jDnDV0spP4ar98LsrLoXuSTiEk/EJJFA8DU4F0M/sVMAmYEtaoROQzH6zdxe3Tili38wCjevfhSv8QfL1VwE9az1EThXPuWTNbDIwh8EzF5c655WGPTKSD27W/hntnLuelT7bQK6Er3ysYyRk+FfCT1nfURGFmf3POfR1Y0cQ0EWlhDQ2OfyzazP2vruBATR3jcvpxqb8f6SrgJxESyqWn/OARr7/i1PCEI9KxLd+6j8lTl/Hxpr0MTunJpPwCFfCTiDvi28/Mfg7cBnQxs30cLuVxEO+ZBhFpGZUH6/jfN1fzxHvrSYiN5fqBJ3OeL4PERJ1CSOQdMVE45+4D7jOz+5xzP2/FmEQ6lDdKtnPnjGI+3VvFWRlZTPQPIitdBfyk7QilM/vnZtaDwBPT8UHT54UzMJFot2VvFXfNKOaNku1kJyXy3yefzqk5PVXAT9qcUDqzvwX8iEAJjiXAaOBDAk9qi8gxqqtv4K/vb+D3b66ivt4x0T+Ii/0+UnroFELaplC6yH5EoFzHfOfcl81sEHBveMMSiU4fb9rD5KlFLN+6j5PT0rnKn8+AjK56JkLatFASRbVzrtrMMLM459wKMxsY9shEokh5ZS0PzF7B8ws20bNLPN8efCpn+3vRtas6q6XtCyVRlJpZMoECfm+Y2R5gY3jDEokOzjmmL/mUX8wsYc+Bg4zJ9DHBP4A+aZ30TIS0G6F0Zk/0Bu8yszlAd+DVsEYlEgXW7dzPlOlFvL+mjH49kvnOiFEMy1YBP2l/jukxHufcO97Zxf8H/Co8IYm0b9W19fxp7lr+OHctnWNO4ur+BVzoz1YBP2m3mnvgLotA8b++BC47PU/g9yi+7g2LSCPvrd7FlOlFrN91gNF9+nKFbzC+PvF6JkLatebOKJ4B3iHwI0VjgUUEbo8d5pzb1gqxibQbOyqq+dXM5Uxf8il9unXl+wWjON2XpgJ+EhWaSxQ9nXN3ecOzzewq4GvOuZB+i0KkI2hocDy7YBO/fm0F1QcbuCS3P5f580hL0f2uEj2a7aPwnsg+dGG1DOhuFrhXwzm3O8yxibRpxZ+WM3lqEUs27yU/NYVJuQUMyeqmAn4SdZp7S3cHFnM4UQB87P3rAH+4ghJpyw7U1PH7N1bx1w820K1zLN8YeDLn+TPo1k2d1RKdmisKmNuKcYi0ec45Xi/Zzl0zitlaXs05Gdlc7h9EVq9YPRMhUU0nySIhKN1TyV0zinlz+Q6ykxL5yfBTODWnB507RzoykfBTohBpRm19A0+8t56H3lyNc3ClfzAX+XNVwE86FCUKkSNYtGE3k6cWsXJ7Baek92KSP59+fbuogJ90OM09cBcP3AL0A5YBTzjn6lorMJFI2Vt5kPtfXcELCzeT2jWe7ww5lbP8venSJdKRiURGc2cUTwO1wLvAxcAQAiXHRaKSc46XPt7Cr2Yup7yqlguz/Vzm668CftLhNZcohjjnhgKY2RPAgtYJSaT1rdmxn9unLWP+ut3075HMLYOGMiw7SQX8RGg+UdQeGnDO1Zm+UkkUqq6t55E5a3j0nbXExcRwbf+hXJCXRVKi3u8ihzSXKIab2T5v2IAu3rgBzjmXFPboRMJo3qqdTJlexMaySk7vk8EV/sHk9o5TAT+RRppLFP9xzo1otUhEWsmOfdXc80oJryzdSt/EBH409DRG5aaqgJ/IETSXKFyrRSHSCuobHM9+tJFfv7aSg3UNXJY7gEvy/KT11P2uIs1pLlGkm9mPjzTTOfdgGOIRCYuiLeVMnrqM/5SWU5CaylW+AgZlJqiAn0gImvsziQG68fmigCLtSkV1LQ++sYqnP9hA97g4vjloOOf6+qqAn8gxaC5RbHXO3dNqkYi0IOccrxZt4+6Xi9mxr4ZzM3MYnzuQrN4q4CdyrJpLFPpzknZp8+5K7phexJyVO8ntnsRPhp/KKSrgJ3LcmksUY0505WY2FniIwGWsx51z9x+h3ZXAi8BI59yiE92udEwH6xp4/L11PPzWanDGpLzBjPXn0iNZ97uKnIjmfo/ihH7BzsxigEeAC4BSYKGZzXDOlTRql0igNMhHJ7I96dgWrN/N7dOWsWr7fgp79eYK3xAV8BNpIeG852MUsMY5tw7AzF4AJgAljdr9AngA+GkYY5EotfvAQe5/dTn/XFRKekIXbhlSyFl5vfRMhEgLCmeiyAA2B42XAqcFNzCzU4As59xMMztiojCzm4GbAbKzs8MQqrQ3zjleXFzKvbOWs6+6jrHZeVyW149eKSrgJ9LSInYXuZmdBDwI3HC0ts65x4DHAAoLC/UgYAe3ensFk6cVsWD9bgb07MH3BhcwNDtJz0SIhEk4/7S2AFlB45netEMSgQJgrldwsDcww8zGq0NbmlJ1sJ7/e3s1j81bR5fYTlw3YChj/CrgJxJIV1nEAAARfklEQVRu4UwUC4H+ZuYjkCCuBq49NNM5Vw6kHho3s7nAT5QkpClzVu7gjmlFbN5TxZf6ZjLRN4gcFfATaRVhSxReafLvA7MJ3B77pHOu2MzuARY552aEa9sSPbbvq+bul4uZtWwbGYkJ/GjoaEblpqizWqQVhfWqrnNuFjCr0bQ7jtD23HDGIu1LfYPjbx9u4Levr+JgXQPjfQMY51cBP5FIUPeftDlLS/cyeWoRy7aUMzQtUMBvYIYK+IlEiv70pM3YV13L72av5Jn5G0mOj+PGQSM419+HhAR1VotEkhKFRJxzjpnLtnLPyyXsrAgU8JvgH0hmugr4ibQFShQSURvLDnDH9GLeWbUTX3IS3xxRyPDsZBXwE2lDlCgkImrq6vnLvHX839triLGTuCpvCGPzcknurlMIkbZGiUJa3fx1ZUyeuoy1Ow8wsncfrvANIa9PvAr4ibRRShTSasr213DvrBX8++NSeiV04bv5IznTn65nIkTaOCUKCbuGBse/Fm/mvlkrqKip4+KcPC7L6096zxh1Vou0A0oUElYrt1Vw+7RlLNywh0E9e/KD/AIKshL1TIRIO6I/VwmLyoN1PPzWGh5/dx1dYzvx9QHDGOPPJFEF/ETaHSUKaXFvr9jOlGnFbNlbxZl9M5noH0x2r84q4CfSTilRSIvZWl7F3TNKeK14G5lJ3bh12GhG5qYQFxfpyETkRChRyAmrq2/gqQ828Ps3VlHX4LjcN5CL/X5Se+oUQiQaKFHICVmyeS+3vbSMkq37GJaWxlW+AgZkdFVntUgU0Z+zHJfyqlp+M3sFz360iR7xcXx78Cmc5eutAn4iUUiJQo6Jc44Z//mUX7yynN0HajgvM5fxvgFkqICfSNRSopCQbdh1gCnTi3h39S78yd351vCRDM/pTmxspCMTkXBSopCjqqmr59G563hk7ho62Ul8tV8+F+Xl0D1JpxAiHYEShTTrgzW7uH1aEet2HWBU7z5c6R+Cr7cK+Il0JEoU0qRd+2v41czlTP1kC70SuvL9glGc7ktTAT+RDkiJQj6nocHxwsLN3P/qcioP1jMupx+X+vuRnqICfiIdlRKFfGb51n1MnrqMjzftZXBKT64qGEp+Vjc9EyHSwekjQDhQU8dDb63miffWkxAby/UDT+Y8X4YK+IkIoETR4b1Rsp07pxfxaXk1Z2dkMdE/iMx0FfATkcOUKDqoLXuruGtGMW+UbCc7KZH/PnkEp+b0VAE/EfkCJYoOpra+gafe38Dv31xFfb1jon8QF/t9pPTQKYSINE2JogP5eNMebntpGSu2VTA8PZ1JvnwGZHTVMxEi0iwlig6gvLKWB2av4PkFm0jpEs+3B5/K2f5edO2qzmoROToliijmnGPaki38cuZy9h6o5fwsH+N9A+iT1knPRIhIyJQootTanfuZMq2ID9aWkdcjme+cUsCwLBXwE5Fjp0QRZapr6/nj3LX8ae5aOsecxDX9C7jAn60CfiJy3JQoosh7q3dx+7RlbCirZHSfvlzhG4yvT7yeiRCRExLWRGFmY4GHgBjgcefc/Y3m/xj4FlAH7ARudM5tDGdM0WhHRTW/mrmc6Us+pU+3BH5QcBqjfakq4CciLSJsicLMYoBHgAuAUmChmc1wzpUENfsEKHTOVZrZd4FfA18NV0zRpqHB8eyCTfz6tRVUH2zg0tz+XOrPIy1F97uKSMsJ5xnFKGCNc24dgJm9AEwAPksUzrk5Qe3nA9eFMZ6oUvxpOZOnFrFk817yU1OY5CtgSKYK+IlIywvnx0oGsDlovBQ4rZn2NwGvNjXDzG4GbgbIzs5uqfjapf01dfz+jVX89f31JMV15oaBw/myvy/duqmzWkTCo018/zSz64BC4Jym5jvnHgMeAygsLHStGFqb4ZxjdvF27n65mK3l1ZyTkc3l/kFk9YrVMxEiElbhTBRbgKyg8Uxv2ueY2fnAZOAc51xNGONpt0r3VHLn9GLeWrGDnO6J/GT4KZya04POnSMdmYh0BOFMFAuB/mbmI5AgrgauDW5gZiOAPwNjnXM7whhLu1Rb38AT763noTdX4xxc6R/MWH8uPVXAT0RaUdgShXOuzsy+D8wmcHvsk865YjO7B1jknJsB/AboBvzLAtdPNjnnxocrpvZk0YbdTJ5axMrtFZzSqxeTfPn069tFBfxEpNWFtY/COTcLmNVo2h1Bw+eHc/vt0d7Kg9z/6gpeWLiZ1K7x3JJfyJm+XnTpEunIRKSjahOd2RLorH7p4y38auZyyqtquSjbz2X+/vROVQE/EYksJYo2YM2O/dw+bRnz1+2mf49kbhk0lGHZSSrgJyJtghJFBFXX1vPInDU8+s5a4mJiuLb/UC7IyyIpUacQItJ2KFFEyDurdjJlWhGbdldyep8MrvAPJrd3nAr4iUibo0TRynbsq+aeV0p4ZelW+iYm8KOhpzEqVwX8RKTtUqJoJfUNjr/P38hvZq/kYF0Dl+UO4JI8P2k9db+riLRtShStoGhLObdNXcbS0nIKUlO5yl/AoIwEFfATkXZBH1VhVFFdy4NvrOLpDzbQPS6OGweN4BxfHxXwE5F2RYkiDJxzvFq0jbtfLmbHvhrOzcxhfO5AsnqrgJ+ItD9KFC1s8+5K7phexJyVO8ntnsRPhxcyIidZBfxEpN1SomghB+sa+Mu76/i/t1djGJP8Qxibl0OPZN3vKiLtmxJFC1iwfjeTpy5j9Y79FPbqzZW+IeSpgJ+IRAklihOw+8BB7pu1nH8tLiU9oQu3DCnkrLxeeiZCRKKKEsVxcM7xr8Wl3DdrOfuq6xibncdlef3olaICfiISfZQojtHq7RVMnlrEgg27GdCzB98bMpShWYl6JkJEopY+3kJUdbCe/3t7NY/NW0eX2E5cN2AoY/wq4Cci0U+JIgRzVu7gjmlFbN5TxZf6ZjLRN4gcFfATkQ5CiaIZ28qrueeVYmYt20ZmUjf+a+hoRvlSiIuLdGQiIq1HiaIJ9Q2OZz7cwO9eX8XBugYm+AYwzp9Hak+dQohIx6NE0cjS0r3cNnUZRVv2MTQtjat8+QxUAT8R6cD08efZV13L72av5Jn5G+kRHyjgd66/DwkJ6qwWkY6twycK5xyvLN3KPa+UsKuihi9n5jLeP4DMdBXwExGBDp4oNpYdYMr0Yuat2okvOYkbRxQyPFsF/EREgnXIRFFTV89j76zjD3PWEGMncVXeEMbm5ZLcXacQIiKNdbhEMX9dGZOnLmPtzgOM7NWHK/1D8PeJVwE/EZEj6DCJomx/DffOWsG/Py6lV0IX/l/+SL7kT1cBPxGRo4j6RNHQ4Pjnos3c9+oKDtTUcXFOHpf6+9MrJUad1SIiIYjqRLFyWwWTpy5j0cY9DErpyVfyC8hXAT8RkWMSlR+ZlQfreOit1Tzx7nq6xnbi6wOGMcafSaIK+ImIHLOoSxRvLd/OHdOL2bK3ijP7ZnJF3mCy0jurgJ+IyHGKmkSxtbyKu2YUM7t4O5lJ3fjxyadTmNNTBfxERE5Qu08UdfUNPPXBBn7/xirqGhyX+wZysd+vAn4iIi2kXSeKJZv3cttLyyjZuo+T09KY5CtgQEZXdVaLiLSgsH6kmtlY4CEgBnjcOXd/o/lxwDPAqUAZ8FXn3Iajrbe8qpbfzF7Bsx9tokd8HN8efApn+XqrgJ+ISBiELVGYWQzwCHABUAosNLMZzrmSoGY3AXucc/3M7GrgAeCrza13b2UtY373DrsP1DAm08cE/wD6pHXSMxEiImESzjOKUcAa59w6ADN7AZgABCeKCcBd3vCLwB/MzJxz7kgr3bynkuxO8VyfP5Jhmd2JjYWamvDsgIhINKirO7Hlw5koMoDNQeOlwGlHauOcqzOzciAF2BXcyMxuBm72Rg/O+fnFa+eEJeT2prYHxO6JdBRtg47FYToWh+lYHHYg53iXbBfdvs65x4DHAMxskXMVhREOqU0IHItqHQt0LILpWBymY3GYmS063mXDeQ/pFiAraDzTm9ZkGzPrBHQn0KktIiJtRDgTxUKgv5n5zKwzcDUwo1GbGcA3vOFJwNvN9U+IiEjrC9ulJ6/P4fvAbAK3xz7pnCs2s3uARc65GcATwN/MbA2wm0AyOZrHwhVzO6RjcZiOxWE6FofpWBx23MfC9AVeRESaozoXIiLSLCUKERFpVptNFGY21sxWmtkaM/tZE/PjzOwf3vyPzCy39aNsHSEcix+bWYmZLTWzt8zsuO+XbuuOdiyC2l1pZs7MovbWyFCOhZl9xXtvFJvZc60dY2sJ4W8k28zmmNkn3t/JuEjEGW5m9qSZ7TCzoiPMNzN72DtOS83slJBW7Jxrcy8Cnd9rAT/QGfgPMKRRm/8HPOoNXw38I9JxR/BYfBno6g1/tyMfC69dIjAPmA8URjruCL4v+gOfAD288fRIxx3BY/EY8F1veAiwIdJxh+lYnA2cAhQdYf444FXAgNHAR6Gst62eUXxW/sM5dxA4VP4j2ATgaW/4RWCMWVRWfDrqsXDOzXHOVXqj8wk8sxKNQnlfAPyCQN2w6tYMrpWFciy+DTzinNsD4Jzb0coxtpZQjoUDkrzh7sCnrRhfq3HOzSNwB+mRTACecQHzgWQz63O09bbVRNFU+Y+MI7VxztUBh8p/RJtQjkWwmwh8Y4hGRz0W3ql0lnNuZmsGFgGhvC8GAAPM7H0zm+9Vc45GoRyLu4DrzKwUmAX8oHVCa3OO9fMEaCclPCQ0ZnYdUAicE+lYIsHMTgIeBG6IcChtRScCl5/OJXCWOc/Mhjrn9kY0qsi4BnjKOfc7MzudwPNbBc65hkgH1h601TMKlf84LJRjgZmdD0wGxjvnorWe7tGORSJQAMw1sw0ErsHOiNIO7VDeF6XADOdcrXNuPbCKQOKINqEci5uAfwI45z4E4oHUVomubQnp86SxtpooVP7jsKMeCzMbAfyZQJKI1uvQcJRj4Zwrd86lOudynXO5BPprxjvnjrsYWhsWyt/INAJnE5hZKoFLUetaM8hWEsqx2ASMATCzwQQSxc5WjbJtmAFc7939NBood85tPdpCbfLSkwtf+Y92J8Rj8RugG/Avrz9/k3NufMSCDpMQj0WHEOKxmA1caGYlQD3wU+dc1J11h3gs/hv4i5ndSqBj+4Zo/GJpZs8T+HKQ6vXH3AnEAjjnHiXQPzMOWANUAt8Mab1ReKxERKQFtdVLTyIi0kYoUYiISLOUKEREpFlKFCIi0iwlChERaZYShXRYZlZvZkuCXrlmdq6ZlXvjy83sziaWyzWzKq9NiZk9Y2axR9lWrpldG769EQkfJQrpyKqcc8ODXhu86e8654YTKIdy3RFKMa/12gwl8HTrV46yrVxAiULaJSUKkSNwzh0AFgP9mmlTDyzAK6zmnTm8a2Yfe68zvKb3A2d5ZyG3mlmMmf3GzBZ6vwvwHW/5PmY2z2tXZGZnhXcvRY6uTT6ZLdJKupjZEm94vXNuYvBMM0shUC/qF0dagZnFA6cBP/Im7QAucM5Vm1l/4HkCZyY/A37inLvUW+5mAuUTRppZHPC+mb0OXAHMds79ysxigK4ttbMix0uJQjqyKu/yUWNnmdknQANwv3OuuIk2eV6S8QEznXNLvemxwB/MbDiBshkDjrDtC4FhZjbJG+9OoGDfQuBJr89jmnNuyRGWF2k1ShQiX/TuoW/+zVjrnBvuFdt738zGezWFbgW2AycTuLR7pB9PMuAHzrnZX5hhdjZwCfCUmT3onHvmuPdEpAWoj0LkBDjndhG4rPRzb1J3YKv3OwdfJ1CkDqCCQBn0Q2YD3z10t5SZDTCzBAv83vl259xfgMcJ/KylSEQpUYicuGlAV6/j+Y/AN8zsP8Ag4IDXZilQb2b/8SqYPg6UAB+bWRGBMvGdCFT+/I936eurwEOtuiciTVD1WBERaZbOKEREpFlKFCIi0iwlChERaZYShYiINEuJQkREmqVEISIizVKiEBGRZv3/ak9h1S3Lh1wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x101a40710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fprs, tprs, thresholds_roc = metrics.roc_curve(y_true, model_1_score)\n",
    "print(\"FP Rates = {}\".format(precisions))\n",
    "print(\"TP Rates = {}\".format(recalls))\n",
    "print(\"thresholds = {}\".format(thresholds_pr))\n",
    "\n",
    "# In matplotlib < 1.5, plt.fill_between does not have a 'step' argument\n",
    "step_kwargs = ({'step': 'post'}\n",
    "               if 'step' in signature(plt.fill_between).parameters\n",
    "               else {})\n",
    "\n",
    "plt.plot(fprs, tprs)\n",
    "plt.fill_between(fprs, tprs, alpha=0.2, color='blue', label=\"Model_1\")\n",
    "\n",
    "plt.xlabel('FP Rates')\n",
    "plt.ylabel('TP Rates')\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.legend()\n",
    "plt.title('ROC curve: AUCROC={:.3f}'.format(metrics.roc_auc_score(y_true, model_1_score)))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Similarly to the PR curve, we only have one threshold $\\left(0.15\\right)$. The threshold $1.15$ is arbitrarily added as $\\max\\left(\\text{y_score}\\right) + 1$.\n",
    "\n",
    "At this threshold, all predictions are positive. Therefore,\n",
    "\n",
    "$$\\text{TPR} = \\frac{\\text{TP}}{\\text{TP} + \\text{FN}} = \\frac{19}{19 + 0} = 1$$\n",
    "\n",
    "$$\\text{FPR} = \\frac{\\text{FP}}{\\text{FP} + \\text{TN}} = \\frac{1}{1 + 0} = 1$$\n",
    "\n",
    "Then, sklearn added an arbitrary point $\\left(0, 0\\right)$ to make sure the curve starts from there (see the [code](https://github.com/scikit-learn/scikit-learn/blob/bac89c253b35a8f1a3827389fbee0f5bebcbc985/sklearn/metrics/ranking.py#L633), where our `tps = [19]` and `fps = [1]`).\n",
    "\n",
    "Eventually, we always can get a line linking $\\left(0,0\\right), \\left(1,1\\right)$ , and its AUC approximation is 0.5.\n",
    "\n",
    "This result holds as long as `y_score` has only one unique value, regardless of the values of `y_true` and `y_score`."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}