ngiyshhk/chapter3.ipynb

## chapter3.ipynb
{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Chapter 3 分類"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## 事前準備\n",
    "```\n",
    "docker run -d -p 8888:8888 jupyter/datascience-notebook\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## 3.1 MNIST"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "* MNISTは、手書きした70000個の数字画像のデータセットです\n",
    "* MNISTは機械学習のHelloWorldです"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# ディレクトリ削除\n",
    "# !rm -rf scikit_learn_data/\n",
    "\n",
    "# mnistデータのフェッチ(動かない)\n",
    "# from sklearn.datasets import fetch_mldata\n",
    "# mnist = fetch_mldata('MNIST original')\n",
    "# mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'COL_NAMES': ['label', 'data'],\n",
       " 'DESCR': 'mldata.org dataset: mnist-original',\n",
       " 'data': array([[0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        ..., \n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),\n",
       " 'target': array([ 0.,  0.,  0., ...,  9.,  9.,  9.])}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://github.com/ageron/handson-ml/issues/143\n",
    "from six.moves import urllib\n",
    "from sklearn.datasets import fetch_mldata\n",
    "\n",
    "from scipy.io import loadmat\n",
    "mnist_alternative_url = \"https://github.com/amplab/datascience-sp14/raw/master/lab7/mldata/mnist-original.mat\"\n",
    "mnist_path = \"./mnist-original.mat\"\n",
    "response = urllib.request.urlopen(mnist_alternative_url)\n",
    "with open(mnist_path, \"wb\") as f:\n",
    "    content = response.read()\n",
    "    f.write(content)\n",
    "mnist_raw = loadmat(mnist_path)\n",
    "mnist = {\n",
    "    \"data\": mnist_raw[\"data\"].T, # インスタンスごとに 1 行、特徴量ごとに 1 列という形の配列を格納する。\n",
    "    \"target\": mnist_raw[\"label\"][0], # ラベルの配列を格納する。\n",
    "    \"COL_NAMES\": [\"label\", \"data\"],\n",
    "    \"DESCR\": \"mldata.org dataset: mnist-original\", # データセットの説明\n",
    "}\n",
    "mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X, y = mnist['data'], mnist['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000, 784)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape # 70,000 個の画像があり、個々の画像には 784 個の特徴量がある"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000,)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAABj5JREFUeJzt3a9rlf8fxvEzGQZZGLo0hA3BWQzivzHEpha1mRRhGkyW\nFUG0WQXFpEFENC6IQWxD0xB/40A4gpyyoJ5P+ZZvuF/3PGdnc+d6POrlvfuAPrnD2/tsot/vd4A8\ne3b6AwA7Q/wQSvwQSvwQSvwQSvwQSvwQSvwQSvwQanKb7+e/E8LoTWzmD3nyQyjxQyjxQyjxQyjx\nQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjx\nQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQyjxQ6jJnf4AMKiHDx+W+5s3\nbxq3+/fvb/XH+T+fPn0a6c/fCp78EEr8EEr8EEr8EEr8EEr8EEr8EMo5PyPV6/Uat5cvX5bXLi8v\nl/urV6/KfWJiotzTefJDKPFDKPFDKPFDKPFDKPFDKEd9Y+7Xr1/lvr6+PtTPbzuO+/DhQ+O2srIy\n1L1HaWZmptzPnDmzTZ9kdDz5IZT4IZT4IZT4IZT4IZT4IZT4IZRz/jHXdo4/Pz9f7v1+v9z/5ddm\njx071ridPXu2vHZxcbHcDx8+PNBn+pd48kMo8UMo8UMo8UMo8UMo8UMo8UMo5/xj7urVq+Xedo7f\ntreZnZ1t3C5cuFBee/369aHuTc2TH0KJH0KJH0KJH0KJH0KJH0KJH0I55x8Dd+/ebdyeP39eXjvs\n+/ht13e73cat7XcKrK2tlfvCwkK5U/Pkh1Dih1Dih1Dih1Dih1Dih1Dih1ATw76v/Ze29WbjojrH\n73Q6naWlpcat1+sNde+d/N7+ubm5cn///v3I7r3LbeovxZMfQokfQokfQokfQokfQokfQjnq2wXa\njry+fv068M+enp4u96mpqXLfs6d+fmxsbDRu379/L69t8/v376GuH2OO+oBm4odQ4odQ4odQ4odQ\n4odQ4odQvrp7Fzh58mS537lzp3E7f/58ee3FixfL/fjx4+XeZn19vXFbXFwsr11dXR3q3tQ8+SGU\n+CGU+CGU+CGU+CGU+CGU+CGU9/kZqW/fvjVuw57z//nzZ6DPFMD7/EAz8UMo8UMo8UMo8UMo8UMo\n8UMo7/P/z5cvX8p93759jduBAwe2+uOMjeqsvu3Xe7ftT548Kfe270FI58kPocQPocQPocQPocQP\nocQPocQPoWLO+W/cuFHu9+7dK/e9e/c2bocOHSqvffz4cbnvZt1ut9yvXbvWuL19+7a8dn5+fpCP\nxCZ58kMo8UMo8UMo8UMo8UMo8UOomKO+169fl/va2trAP/vz58/lfuXKlXK/devWwPcetbZXnZ89\ne1bu1XHe5GT9z+/o0aPl7pXd4XjyQyjxQyjxQyjxQyjxQyjxQyjxQ6iYc/5Rmp6eLvd/+Ry/zeXL\nl8u97euzK7OzsyP72bTz5IdQ4odQ4odQ4odQ4odQ4odQ4odQMef8bV8DPTU1Ve69Xq9xO3HixCAf\naVucPn263B89elTu/X6/3Nt+jXbl5s2bA1/L8Dz5IZT4IZT4IZT4IZT4IZT4IZT4IVTMOf/t27fL\n/d27d+VefT/9xsZGeW3bWXqb5eXlcv/582fj9uPHj/LatnP6I0eOlPu5c+cG3vfv319ey2h58kMo\n8UMo8UMo8UMo8UMo8UOoibZXNrfYtt7sb6ysrJT70tJS41a97tvpdDofP34s91G+NruwsFDuMzMz\n5f7gwYNyn5ub++vPxMht6h+MJz+EEj+EEj+EEj+EEj+EEj+EEj+Ecs6/Sd1ut3Fre212dXW13F+8\neFHuT58+LfdLly41bqdOnSqvPXjwYLmzKznnB5qJH0KJH0KJH0KJH0KJH0KJH0I554fx45wfaCZ+\nCCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+\nCCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CCV+CDW5zfeb\n2Ob7AQ08+SGU+CGU+CGU+CGU+CGU+CGU+CGU+CGU+CGU+CGU+CGU+CGU+CGU+CGU+CGU+CGU+CGU\n+CGU+CGU+CGU+CGU+CGU+CHUf5Zt+b+OQHReAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f81dcfa70f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "some_digit = X[36000]\n",
    "some_digit_image = some_digit.reshape(28, 28)\n",
    "\n",
    "plt.imshow(some_digit_image, cmap = matplotlib.cm.binary, interpolation='nearest')\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[36000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# データを詳しく調べる前に、テストセットを作って封印\n",
    "# 訓練セット(最初の 6 万画像)とテストセット(後ろの 1 万画像)\n",
    "X_train, X_test, y_train, y_test = X[:60000], X[60000:], y[:60000], y[60000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([34574, 42277, 51783, ..., 20347, 55811, 34579])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "shuffle_index = np.random.permutation(60000)\n",
    "shuffle_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X_train, y_train = X[shuffle_index], y[shuffle_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## 3.2 二項分類器の訓練"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y_train_5 = (y_train == 5)\n",
    "y_test_5 = (y_test == 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, ..., False, False, False], dtype=bool)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### scikit-learn の SGDClassifier クラスを使って、確率的勾配降下法(SGD)の分類器から試してみる"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "* 非常に大規模なデータセットを効率よく扱えるという長所\n",
    "* オンライン学習にも向いている"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SGDClassifier(alpha=0.0001, average=False, class_weight=None, epsilon=0.1,\n",
       "       eta0=0.0, fit_intercept=True, l1_ratio=0.15,\n",
       "       learning_rate='optimal', loss='hinge', n_iter=5, n_jobs=1,\n",
       "       penalty='l2', power_t=0.5, random_state=42, shuffle=True, verbose=0,\n",
       "       warm_start=False)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "sgd_clf = SGDClassifier(random_state=42)\n",
    "sgd_clf.fit(X_train, y_train_5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True], dtype=bool)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## 3.3 性能指標"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### 3.3.1 交差検証を使った正解率の測定"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.94775,  0.96575,  0.93465])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "cross_val_score(sgd_clf, X_train, y_train_5, cv=3, scoring='accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.9108 ,  0.9118 ,  0.90635])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.base import BaseEstimator\n",
    "\n",
    "class NeverSClassifier(BaseEstimator):\n",
    "    def fit(self, X, y = None):\n",
    "        pass\n",
    "    def predict(self, X):\n",
    "        return np.zeros((len(X), 1), dtype=bool)\n",
    "\n",
    "never_5_clf = NeverSClassifier()\n",
    "cross_val_score(never_5_clf, X_train, y_train_5, cv=3, scoring='accuracy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "いつも５ではないと予想し続けるモデルでも90%は当たる(正解率が指標として好まれない理由)  \n",
    "特に歪んだデータ（一部のデータの出現頻度が多いデータ）では正解率はあてにならない"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### 3.3.2 混同行列"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "* クラス A のインスタンスがクラス B に分類された回数を数えるというもの\n",
    "* 実際のターゲットと比較できる予測の集合をcross_val_predict()で作る"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_predict\n",
    "y_train_pred = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[52341,  2238],\n",
       "       [  799,  4622]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "confusion_matrix(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "* 各行は実際のクラス、各列は予測したクラス\n",
    "* ([真陰性(正しく5以外と分類), 偽陽性(誤って5と分類)], [偽陰性(誤って5以外と分類), 真陽性(正しく5と分類)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### 適合率\n",
    "* TP/(TP+FP)\n",
    "* TP: 真陽性の数, FP: 偽陽性の数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### 再現率\n",
    "* TP/(TP+FN)\n",
    "* FN: 偽陰性の数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### 3.3.3 適合率と再現率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.67376093294460637"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score\n",
    "precision_score(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.85261021951669436"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### F値\n",
    "* 適合率と再現率の調和平均\n",
    "* TP/(TP+(FN+FP)/2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.75270743424802544"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "f1_score(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "* 適合率の方が重視される場合(子どもに見せても安心なビデオを検出する分類器を訓練している場合)\n",
    "* 再現率の方が重視される場合(監視ビデオか ら万引き犯を見つける分類器を訓練している場合)\n",
    "* 適合率と再現率はトレードオフの関係にある"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### 3.3.4 適合率と再現率のトレードオフ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 129202.99030501])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_scores = sgd_clf.decision_function([some_digit])\n",
    "y_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True], dtype=bool)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threshold = 0\n",
    "y_some_digit_pred = (y_scores > threshold)\n",
    "y_some_digit_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False], dtype=bool)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# しきい値をあげる\n",
    "threshold = 200000\n",
    "y_some_digit_pred = (y_scores > threshold)\n",
    "y_some_digit_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y_scores = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3, method='decision_function')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "precisions, recalls, thresholds = precision_recall_curve(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEKCAYAAADpfBXhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VFX6wPHvOwlJIPTA0oKhl9AChK50EJamdGxYdnVV\nwLK6FnaVtS7KWnBRLOsPdFkBQV1UlAiCSJUiNbTQA0gnBEJIO78/7gRCSJkkM7kzk/fzPPPMvfec\nuee9yeSdmzPnnivGGJRSSvkXh90BKKWUcj9N7kop5Yc0uSullB/S5K6UUn5Ik7tSSvkhTe5KKeWH\n8k3uIvKxiJwQkW25lIuITBWROBHZIiJt3B+mUkqpgnDlzH0G0C+P8v5AQ+fjfuC9ooellFKqKPJN\n7saY5cCZPKoMAT4xljVARRGp4a4AlVJKFVygG/ZRCzicZT3eue1Y9ooicj/W2T2hoaFtmzRpUuDG\njiYe5VjidbsuUUTEeiBXtyFkrmbfLiI4xGE94yDAEXBl2eFw4MBxZX8OsbZlLgc6Aq+UZd9X1m0i\nQoAEXFlXyt0SEyE9HUqVguBgCMyWvS5fhrNnISwMkpIgLu76fTRsCOXLW/X27bu+vEkTCA2FU6fg\n4MHry5s1g5AQOHECDh++vrxFCwgKgmPH4OjR68ujoiAgwLXjzc2GDRtOGWOq5lfPHck9p7/kHOc0\nMMZ8AHwAEB0dbdavX1/gxowx7D+3n7SMtCvrVxs1uW5zZXtB6rqjPVf2cebSGc4ln+NU0imS05K5\nmHKRi6kXMcaQYTLIMBkYsiznsD05LZlLaZdISU/hUuolzl8+z+X0yySnJV9ZTs1IJTU9ldSMVNyh\ndGBpQoNCCQoIomxQWcLLh1O5dGWCAoIIdAQSKIEEOgIpU6oMIYEh1jbnIzQolIohFaletjo1ytag\nVvlahJUO0w+NEiIjA264AZ58Eh55BE6ehFdegbfeulrnrbessrg4GDoUmjaFQ4dg2zaYPx9atbIS\ndHZVqljJOSkJzuTQH1G1qvXBcfGi9QGQXbVq1odLYiIkJFxfXr269aFz/rz1yK5mTXAUcRiLiOTw\nsXM9dyT3eKB2lvVwIIfPLPcQEepVquep3Zd4KekpJKclk5KewvnL5zl76SxpGWlXHqkZqSReTrxm\nW1pGGhdTL155XeZrL6dd5nL6ZU5cPMGppFNsPb+VdJN+dV/pqSSmXN1XhsnIM7bggGBqlKtB47DG\nVAypSLXQatQsV5Ma5WpQtUxVIipGULdiXUqXKl1MPy3lbikp8Le/wZEjsHKllcB37Lia2KdMsRJ5\n5j/96ekQEQHr1sH+/da26tWtBB4enns7ZcpYj9yEhlqP3JQrZz1yU7689bCTO5L7AmCciMwGOgAJ\nxpiS3W/iw4ICgggKCAKgSpkqUKn42s4wGaRnpJNwOYGzl84Sfz6eU0mniD8fT8LlBJJSk9hzZg9H\nE48SdyaO4xePcyHlwnX7uaHCDTT/XXNqlK1BheAK1KlYh7qV6tIxvKN1TMrtFi+GChWudls4HFY3\nyZw5UKuWdTbevLmVqHv2hFmzICYGKlWyHidOwLRpUL8+xMdb+3z1Veu5a1dIToYLF6wul6waN4av\nv7aWk5KsRxX9FQMuJHcR+QzoDlQRkXjgeaAUgDFmOrAQ+D0QByQB93gqWOXfHOLAEeCgSpkqVClT\nhYZhDfOsb4zhYupFjiUe41DCIeLPx7P/3H42/baJA+cO8OuxXzmbfJbktOQrr6lRtgaNwhoRVT2K\nltVaXvkgqF62uqcPzy8kJV09o61QAd59F8aMgT59rq03dixMmgQLF1oJPqtly+Cbb2D27KvbataE\nP/3JSvyffAKbN1uJPlNwsPXIS35n4yWN2DXlb2H73JUqCGMMxy8eJ/ZkLGvi17DnzB52ndrFpt82\ncSnt0pV6zX/XnGFNhzG06VBaVmtpY8TeLaevPTIyrCS+caP1fOmSdTb+yCPQoQOsXg1r1ljdGDEx\nsHy51d+dnAzbt1sfEufPQ69eRe+PLglEZIMxJjrfeprcVUmUnpHOgXMHiDsTx4ZjG/hq51esO7oO\ngJrlatIorBFdandhROQIWlZrWaK/zDUGRoywzqo7dIDoaKuvOyXFGvlRqpTdEZYsmtyVKqATF0/w\nny3/YfPxzcSejGX9Uev9WadiHe5qeRd96vehdfXWhAbl8U2bH/ngA3jggWu3paUVfSifKhpN7koV\n0ZHzR5izfQ5zt89l7ZG1AARIAG1rtmV0s9Hc2/peKoRUsDlK90pMhOeft/rLx42DTz+1tjdrBnPn\nQmSkreEpNLkr5VYnL55kdfxqVhxawVc7v2LPmT2EBIZwZ8s7eaLzEzQKa2R3iEXWqxf8+KO1PG8e\ntGlj9Ys3bWpvXOpamtyV8pD0jHRWHV7F+xveZ9bWWQA81eUpnuv2HGVK+d5wjdRU6NLFGisOMHIk\n/Pe/2v3irVxN7vrdtFIFFOAI4KaIm/jP0P8QNz6Okc1GMnnlZJq924w3V7955eppXxEQYI1JB+uq\nzjlzNLH7A03uShVB/cr1mT1sNl+N+gpjDI/HPE7ktEg2HN1gd2j5OnsWate2LqOPibFGxWS/SEj5\nLk3uShWRiDCkyRAOPHqAT2/9lDOXztD+o/a8tPwl7Or2zM/atVC5snU16IgR1pwpyr9oclfKje5o\neQc7x+3k1ia38relf6PbjG4cOX/E7rCusXkzdOxoLb/2mjV1gPI/mtyVcrMqZaowd8Rc3rz5TdYe\nWUvz95ozL3ae3WFdMXWq9TxunDXzovJPmtyV8gCHOHi046Os/cNa6lWqx4jPR/BkzJO2d9OkpcH0\n6bBpE7zzjq2hKA/T5K6UB0VVj2LVvavo16AfU1ZP4U/f/CnfqY095eabrfnKMzKs+c6Vf9PkrpSH\nBQcG882Yb7i/zf18sPEDJnw3odhjePJJa0TMuXM532RC+R93zOeulMpHgCOA6QOnk2EymLZuGpVL\nV+bv3f9eLBOSJSRYN7kAa/hjxYoeb1J5AU3uShUTEeG9ge9x+tJpXlz+Ihkmgxd7vOjxBP/cc9bz\nI49oYi9JtFtGqWIU6Ahk3sh5jIgcwcs/v8z478Z7vM3u3aFBA3jzTY83pbyIJneliplDHMwePptb\nm9zKtHXTWLhnocfa2r4dhgyBPXtyvtGG8l+a3JWygUMcfHrrp0RWjWTk5yPZcnyL29uIjbUmBFux\nwu27Vj5Ak7tSNgkNCiXmjhhEhLu+vIuk1CS37fu336w52BMSrC4ZVfJoclfKRrXK1+KjQR+x+fhm\nZm6a6ZZ9GnP1phpPPWXdfFqVPJrclbLZyGYjaVqlKW+vfZvktOQi7++tt6whj507wz/+4YYAlU/S\n5K6UzUSESd0nsev0Lt5ZW/Q5ATp0sL48XbrUDcEpn6XJXSkvMLLZSPo16Mfzy54v0iySSUnWGXtG\nBgQFuTFA5XM0uSvlJSb3nsyltEu8tPylQr3+8mWIioI33nBzYMonaXJXyku0rNaSB6MfZPqG6aw7\nsq7Ar3/3XWs8e5L7Bt0oH6bJXSkv8mKPFykXVI4J308gNT21QK+dNs16fuopDwSmfI4md6W8SFiZ\nMN7q9xZr4tfw4vIXXX7dmjWwdy+88AKUKuXBAJXP0OSulJe5t/W9DGw0kHd+eYeEZNfm533/fev5\nj3/0YGDKp2hyV8oL/b373zmXfI4pq6a4VP+xx+Bvf4Pq1T0cmPIZmtyV8kJtarRhWNNh/Gvdv/I9\ne1+92hop88ILxRSc8gma3JXyUk/f+DTnks/lOTRyzx5rXPujjxZjYMonaHJXyktF14zmlia3MGvr\nrFzvu/qHP1jPt99ejIEpn6DJXSkvNrzpcI5dOMba+LXXlWVkwLZt0LQpPPSQDcEpr6bJXSkvNrDR\nQIICgpgXO++6sgcfhDNnYOJEGwJTXk+Tu1JerEJIBfrW78vs7bNJz0i/pmzUKOt5yBAbAlNez6Xk\nLiL9RGSXiMSJyNM5lN8gIktF5FcR2SIiv3d/qEqVTPdE3cPRxKN8Hvv5Ndt79rTmbi9b1qbAlFfL\nN7mLSAAwDegPRAJjRCQyW7W/AnONMa2B0cC77g5UqZLqlia30CisEe+tf+/KtunTYcECG4NSXs+V\nM/f2QJwxZp8xJgWYDWT/R9AA5Z3LFYCj7gtRqZLNIQ5ua34bPx/8mSPnj2CM1d+u3TEqL64k91rA\n4Szr8c5tWU0C7hCReGAhMD6nHYnI/SKyXkTWnzx5shDhKlUyjW4+GoPhvfXvsWGDte2ZZ+yNSXk3\nV5K75LDNZFsfA8wwxoQDvwc+FZHr9m2M+cAYE22Mia5atWrBo1WqhGpcpTHta7Xn+7jvmTPHmhzs\nySftjkp5M1eSezxQO8t6ONd3u9wHzAUwxqwGQoAq7ghQKWUZETmCDcc2MOv7XfTpA5Uq2R2R8mau\nJPd1QEMRqSsiQVhfmGb/KucQ0AtARJpiJXftd1HKjUY3H41DHJyL+OTKMEilcpNvcjfGpAHjgEXA\nDqxRMdtF5AURGeys9mfgjyKyGfgMuNsYk73rRilVBOHlw+lTrw8Ve/wfI0am5/8CVaIFulLJGLMQ\n64vSrNuey7IcC3Rxb2hKqayMgbGt7ua2L8aw+thP9Kzb0+6QlBfTK1SV8hGrV8MzwwZRoVTla8a8\nK5UTTe5K+Yi//hUO7gnljhZj+XLHl8Sfj7c7JOXFNLkr5QPS0mDpUmv5iZsmAPD2mrdtjEh5O03u\nSvmAFSus588+gzoV6zC48WBmbJ5BclqyvYEpr6XJXSkfMH8+lC4NgwZZ6+Paj+NU0ik+3/553i9U\nJZYmd6V8wPDhMGUKhIZa6z3q9KB2+drM3zHf3sCU19LkrpQP6Nbt2rstiQhDGg8hZm8MSalJ9gWm\nvJYmd6W83LRpOd8Ae0iTIVxKu0TM3pjiD0p5PU3uSnm5cePg7RwGxnSL6EalkEo53oJPKU3uSnmx\n336znu+55/qyUgGlGNhoIN/s/oaU9JTiDUx5PU3uSnmxTz6xnv/855zLRzUbRcLlBL7d/W3xBaV8\ngiZ3pbzYU09Zz82a5Vx+c4ObKR9cnnk7tGtGXculicOUUsUvPR1q1oSbbsq9TqAjkKFNhzIvdh4X\nUi5QNkjvlq0seuaulJcKCIAjR6yrUvNyb9S9XEi5wPxYHfOurtLkrpSXyrwjguR0o8ssbrzhRupU\nrKMXNKlraHJXyks5HNC/f/71RISBDQeyeN9izl466/nAlE/Q5K6UFzrpvEnl+fOu1b+r1V1cSrvE\n/3b9z3NBKZ+iyV0pL/S5cz6wyZNdqx9dM5rw8uF8HqsTiSmLJnelvNCmTdZz+/au1RcRhjUdxo/7\nf9S5ZhSgyV0pr/Thh9ChAwQFuf6aQY0GkZyWzNe7vvZcYMpnaHJXyssYA0OHwrBhBXtdj7o9qF62\nuo6aUYBexKSU1xGxbs5RUA5xMKjRIGZtncXltMsEBwa7PzjlM/TMXSkv8/zzMHFi4V47oOEAklKT\nWHFohXuDUj5Hk7tSXua99+CVVwr32l71euEQB0sPLHVvUMrnaHJXyoskJkJCAjz5ZOFeXzaoLM2q\nNuOXI7+4NzDlczS5K+VFliyBlBQYMKDw++hZtyc/H/qZ5LRk9wWmfI4md6W8yM8/Q3CwNQyysHrX\n601yWjKrDq9yX2DK52hyV8qL7NgBHTtCSEjh99EtohuBjkB+2PuD+wJTPkeTu1Je5Ntv4ZtviraP\ncsHl6BjekR/2aXIvyTS5K+VFRKCsG+630adeHzYe28ixxGNF35nySZrclfISkyfDHXdcnce9KG5r\ncRsA09ZNK/rOlE/S5K6Ul/jySzhwIP+bc7iiQeUG9K7Xm7nb5xZ9Z8onaXJXygskJsK6ddC9u/v2\neWuTW9lzZo+OeS+hNLkr5QW+/hoyMqBdO/ft8/aWt1PKUYp5sfPct1PlM1xK7iLST0R2iUiciDyd\nS52RIhIrIttF5L/uDVMp/7Z7t/Xco4f79lk+uDwdwzuycM9C9+1U+Yx8k7uIBADTgP5AJDBGRCKz\n1WkIPAN0McY0Ax71QKxK+a2ICBg9GsqXd+9+b2lyC9tPbmf36d3u3bHyeq6cubcH4owx+4wxKcBs\nYEi2On8EphljzgIYY064N0yl/Ns998Bnn7l/v6Obj0YQZm2Z5f6dK6/mSnKvBRzOsh7v3JZVI6CR\niKwUkTUi0i+nHYnI/SKyXkTWn8y8A7BSJVxSEiR7aBqYmuVq0jWiK3O2z8G4Y4yl8hmuJPecBmZl\nf5cEAg2B7sAY4CMRqXjdi4z5wBgTbYyJrlq1akFjVcovzZ4NFSvCoUOe2f/wyOHsOr2LfWf3eaYB\n5ZVcSe7xQO0s6+HA0Rzq/M8Yk2qM2Q/swkr2Sql8TJkCly9DeLhn9t+rbi8Aftz/o2caUF7JleS+\nDmgoInVFJAgYDSzIVucroAeAiFTB6qbR0wSlXLBjh/Xs8NDA5CZVmlCzXE2da6aEyfftZIxJA8YB\ni4AdwFxjzHYReUFEBjurLQJOi0gssBR40hhz2lNBK+Uv0tKsuWQeeshzbYgIfev3Zcn+JWSYDM81\npLyKS+cKxpiFxphGxpj6xpiXndueM8YscC4bY8zjxphIY0wLY8xsTwatlL/YtAkuXICbbvJsOz3q\n9ODMpTNsPb7Vsw0prxFodwBZpaamEh8fT7Knhg74uZCQEMLDwylVqpTdoSgX1axp9bm78+KlnPSs\n2xOA7+O+p1X1Vp5tTHkFsWt4VHR0tFm/fv012/bv30+5cuUICwtD3DF7UglijOH06dMkJiZSt25d\nu8NRXqjtB20JdASy9g9r7Q5FFYGIbDDGROdXz6vmlklOTtbEXkgiQlhYmP7X42O+/x5++6142hrW\ndBi/HPmFkxf1GpOSwKuSO6CJvQj0Z+dbTp2C/v1h4sTiaa97ne4ALNq7qHgaVLbyuuRut4CAAKKi\nomjevDkjRowgKSmpyPtcv349EyZMyLX86NGjDB8+vMjtKN+yerX1fMstxdNeh1odqF62uk4kVkJo\ncs+mdOnSbNq0iW3bthEUFMT06dOvKTfGkJFRsOFk0dHRTJ06NdfymjVrMm+eTsta0qxaBYGB0Lt3\n8bQX4AigW0Q3lh1YplMRlACa3PNw0003ERcXx4EDB2jatCkPPfQQbdq04fDhw8TExNCpUyfatGnD\niBEjuHDhAgDr1q2jc+fOtGrVivbt25OYmMiyZcsYOHAgAD/99BNRUVFERUXRunVrEhMTOXDgAM2b\nNwes7x3uueceWrRoQevWrVm6dCkAM2bMYOjQofTr14+GDRvyl7/8xZ4finKb1ashKgpKly6+NvvW\n78uxC8f49bdfi69RZQuvGgqZXU53pRk50rrgIykJfv/768vvvtt6nDoF2Xs6li1zve20tDS+++47\n+vWz5kDbtWsX//d//8e7777LqVOneOmll1i8eDGhoaFMnjyZN954g6effppRo0YxZ84c2rVrx/nz\n5ymd7S93ypQpTJs2jS5dunDhwgVCQkKuKZ82zbrn5datW9m5cyd9+/Zlt3Oy702bNvHrr78SHBxM\n48aNGT9+PLVr10b5npQU+OUXuP/+4m03c0jkqsOraFOjTfE2roqVnrlnc+nSJaKiooiOjuaGG27g\nvvvuAyAiIoKOHTsCsGbNGmJjY+nSpQtRUVHMnDmTgwcPsmvXLmrUqEE75+10ypcvT2DgtZ+fXbp0\n4fHHH2fq1KmcO3fuuvIVK1Zw5513AtCkSRMiIiKuJPdevXpRoUIFQkJCiIyM5ODBgx79WSjPCQqC\nnTvh8ceLt92IChFEVIjQeWZKAK8+c8/rTLtMmbzLq1Qp2Jl6psw+9+xCQ0OvLBtj6NOnD59lm4B7\ny5Yt+Y5YefrppxkwYAALFy6kY8eOLF68+Jqz97z6QoODg68sBwQEkJaWlu/xKO91ww3F36aI0KNu\nDxbsWkCGycAhen7nr/Q3WwgdO3Zk5cqVxMXFAZCUlMTu3btp0qQJR48eZd26dQAkJiZel4D37t1L\nixYteOqpp4iOjmbnzp3XlHft2pVZs6wbK+zevZtDhw7RuHHjYjgqVZxef92a6tcOPev05MylM2z6\n7fqTGOU/NLkXQtWqVZkxYwZjxoyhZcuWdOzYkZ07dxIUFMScOXMYP348rVq1ok+fPtddVPTWW2/R\nvHlzWrVqRenSpenfv/815Q899BDp6em0aNGCUaNGMWPGjGvO2JXvS0uDv/wF5syxp/3M8e4rDq2w\nJwBVLLxq+oEdO3bQtGlTW+LxF/oz9H4bNkB0NIwfD3mMkPWoWm/Uonud7swaqrff8zU+Of2AUiXB\nCucJs52jWTvU6sCa+DX2BaA8TpO7UsVs5Urry1RP3XnJFR1qdWDf2X3sPr3bviCUR2lyV6qYBQRA\n3772xjA2aiwOcfDp5k/tDUR5jCZ3pYrZZ5/Bhx/aG0P1stVpX6s9i/cvtjcQ5TGa3JUqRt40pcuA\nhgNYE7+GwwmH7Q5FeYAmd6WKUb9+UKuW3VFYRjcfDcDsbXpXTH+kyT2brFP+Dho0iHPnzrl1/zNm\nzGDcuHEATJo0iSlTprh1/8q7xcTA0aN2R2FpULkBUdWj+HbPt3aHojxAk3s2Waf8rVy58pWJvJQq\nqpQUcDjgscfsjuSqrjd0Zd3RdaSmp9odinIzTe556NSpE0eOHLmy/vrrr9OuXTtatmzJ888/f2X7\nJ598QsuWLWnVqtWVSb++/vprOnToQOvWrenduzfHjx8v9viVd9mwATIyoEsXuyO5qkfdHiSlJulE\nYn7IaycOe/T7R90+90VU9Sje6veWS3XT09NZsmTJlVkhY2Ji2LNnD7/88gvGGAYPHszy5csJCwvj\n5ZdfZuXKlVSpUoUzZ84AcOONN7JmzRpEhI8++ojXXnuNf/7zn249HuVbMi9euvFGe+PIql+DflQp\nU4UPN37IzQ1utjsc5UZem9ztkjnl74EDB2jbti19+vQBrOQeExND69atAbhw4QJ79uxh8+bNDB8+\nnCpVqgBQuXJlAOLj4xk1ahTHjh0jJSWFunXr2nNAyms0bw4TJkC1anZHclVIYAi3Nb+N99a/x+mk\n04SVCbM7JOUmXpvcXT3DdrfMPveEhAQGDhzItGnTmDBhAsYYnnnmGR544IFr6k+dOjXHaX7Hjx/P\n448/zuDBg1m2bBmTJk0qpiNQ3qp/f+vhbcZGjWXqL1NZsGsB97S+x+5wlJton3suKlSowNSpU5ky\nZQqpqancfPPNfPzxx1dup3fkyBFOnDhBr169mDt3LqdPnwa40i2TkJBALeeYt5kzZ9pzEMprJCTA\noUPeNc49U+vqrakWWo0f9v1gdyjKjTS556F169a0atWK2bNn07dvX2677TY6depEixYtGD58OImJ\niTRr1oyJEyfSrVs3WrVqxePOW+tMmjSJESNGcNNNN13pslEl18KFEBEBW7bYHcn1RIT+DfvzXdx3\npGXoDWD8hU7562f0Z+idHn4YPvkEzpyBUqXsjuZ6c7fPZdS8Uay+bzUdwzvaHY7Kg075q5QX+fFH\na5SMNyZ2gF51eyEIMXtj7A5FuYkmd6U87LffrJth9+5tdyS5CysTRtuabbXf3Y9oclfKwzLHt3vT\nxUs56VOvD2vi13D+8nm7Q1Fu4HXJ3a7vAPyB/uy8U58+MGMGtGljdyR5612vN2kZaaw8tNLuUJQb\neFVyDwkJ4fTp05qkCsEYw+nTpwkJCbE7FJVNhQowdiwEBdkdSd7a1mgLwIZjG2yORLmDV13EFB4e\nTnx8PCdPnrQ7FJ8UEhJCuJ33blPXOXEC5syBYcOgZk27o8lbhZAKRFaNZPnB5XaHotzAq5J7qVKl\n9DJ95Ve++86acqBrV+9P7gCDGw3m9VWvc/LiSaqGVrU7HFUELnXLiEg/EdklInEi8nQe9YaLiBGR\nfMdgKlUSLFkCVatCixZ2R+KaMS3GkG7S+WLHF3aHoooo3+QuIgHANKA/EAmMEZHIHOqVAyYAa90d\npFK+yBj44Qfo2dOax90XtPhdC5pUacLs7Xp3Jl/nyluuPRBnjNlnjEkBZgNDcqj3IvAakOzG+JTy\nWZs3W2Pc+/WzOxLXiQijmo3ipwM/cSzxmN3hqCJwJbnXArLeQTfeue0KEWkN1DbGfJPXjkTkfhFZ\nLyLr9UtT5e9iY60zdm++eCkntza5FYPRC5p8nCvJ/fr5bOHKWEURcQBvAn/Ob0fGmA+MMdHGmOiq\nVfXLGuXfbrsNTp8GXxvA1KJaC8JKh7Fk/xK7Q1FF4EpyjwdqZ1kPB7Le4rcc0BxYJiIHgI7AAv1S\nVSmoWNHuCArOIQ561evF4n2L9ZoTH+ZKcl8HNBSRuiISBIwGFmQWGmMSjDFVjDF1jDF1gDXAYGPM\n+px3p5T/+/57a6KwgwftjqRwetftzdHEo+w8tdPuUFQh5ZvcjTFpwDhgEbADmGuM2S4iL4jIYE8H\nqJQv+vZb+PVXqF7d7kgKp0996/aS2u/uu1waoGWMWWiMaWSMqW+Medm57TljzIIc6nbXs3ZVkhlj\nXbzUsycEB9sdTeHUqViH+pXqs3jfYrtDUYXkI6NvlfIdy5bB3r3eeb/UguhVtxfLDy4nPSPd7lBU\nIWhyV8rNvv/eeh4xwt44iqp7ne4kXE5g3dF1doeiCkGTu1Ju1rEjPPaYNe2AL+tbvy8OcbBwz0K7\nQ1GFoMldKTe79VZ44w27oyi6sDJhdAzvyLd7vrU7FFUImtyVcqO4ODh+3O4o3GdQo0FsPLaR+PPx\ndoeiCkiTu1Ju9Ne/Wndc8pdrf4Y0tqaR+t/O/9kciSooTe5KuUlaGixaZN1WT3KatMMHNa3alMiq\nkXwe+7ndoagC0uSulJusWgXnzsHAgXZH4l7Dmw5n+cHlHE44nH9l5TU0uSvlJgsWWPdJvflmuyNx\nrzta3oHBMH/HfLtDUQWgyV0pN1m4ELp1g3Ll7I7EvRqGNaTF71pocvcxmtyVcpOYGHj9dbuj8Izh\nkcNZeWil3sDDh2hyV8pNwsOhVSu7o/CMYU2HYTB8ufNLu0NRLtLkrpQbjBtnTRbmryKrRtKkShMd\nNeNDNLlG6GNyAAAUz0lEQVQrVUS7dsG0adYFTP5KRBjUaBArD63kQsoFu8NRLtDkrlQRffGF9XzL\nLfbG4Wm96/UmNSOVlYdW2h2KcoEmd6WKwBiYNQs6dYLatfOv78s6hXeilKOU3lvVR2hyV6oINm2C\n7dvhrrvsjsTzygWXo3PtzprcfYQmd6WK4OxZay6ZkSPtjqR4dI3oyubfNnP+8nm7Q1H50OSuVBH0\n7AkbNkDlynZHUjy6RXQj3aSz4tAKu0NR+dDkrlQhHTpkzSVTknS5oQtlSpXh611f2x2Kyocmd6UK\n6c47oUkTyMiwO5LiExIYwqBGg5i3Yx5pGWl2h6PyoMldqUI4fhx+/hk6dwZHCfsrGt18NKeSTvHD\n3h/sDkXloYS9LZVyj48+soZBvvqq3ZEUv/4N+lMhuIJererlNLkrVUCpqfD++9CrFzRubHc0xS84\nMJgBjQbw9e6vtWvGi2lyV6qAli6Fw4fhkUfsjsQ+Q5sM5VTSKR0148U0uStVQH37wsaNMGCA3ZHY\np1+DfoQEhvDFji/sDkXlQpO7UgWQeePr1q1L3hepWYUGhdKvQT++2PEFGaYEDRfyISX47alUwQ0c\nCE89ZXcU3mFok6EcSTzC+qPr7Q5F5UCTu1IuWrnSupVetWp2R+IdBjYaSKAjkC936A08vJEmd6Vc\n9OKLULUqPPCA3ZF4h0qlK9GjTg/m75iPyeyvUl5Dk7tSLli7FhYtgscfh9BQu6PxHkObDmXPmT3E\nnoy1OxSVjSZ3pVzwzDNWd8zDD9sdiXcZ0ngIguioGS+kyV0pF3z8MXz2GZQrZ3ck3qVGuRp0rt2Z\n2dtna9eMl9HkrlQeLl+2hj/WqQM9etgdjXca22ossSdjWRO/xu5QVBYuJXcR6Sciu0QkTkSezqH8\ncRGJFZEtIrJERCLcH6pSxe/hh2HQoJI182NBjWo+isqlK/PSzy/ZHYrKIt/kLiIBwDSgPxAJjBGR\nyGzVfgWijTEtgXnAa+4OVKnitmoV/PvfEBlZsi9Yyk/54PI83O5hvtvzHScvnrQ7HOXkylu2PRBn\njNlnjEkBZgNDslYwxiw1xiQ5V9cA4e4NU6nilZoKXbpAeDg895zd0Xi/YU2HYTDM3zHf7lCUkyvJ\nvRZwOMt6vHNbbu4DvsupQETuF5H1IrL+5En9hFfe68EHrefXXoOyZe2NxRe0rNaS8PLhxOyNsTsU\n5eRKcpcctuX4tbiI3AFEA6/nVG6M+cAYE22Mia5atarrUSpVjJKTYdky6/6oY8bYHY1vEBEGNhxI\nzN4Yzl46a3c4CteSezxQO8t6OHA0eyUR6Q1MBAYbYy67Jzylil9IiHXR0v/+Z3ckvuX+tvdzMfUi\nH238yO5QFK4l93VAQxGpKyJBwGhgQdYKItIaeB8rsZ9wf5hKeV5aGrzzjjX8MSxMu2MKqnWN1nQK\n78TMzTN1zLsXyDe5G2PSgHHAImAHMNcYs11EXhCRwc5qrwNlgc9FZJOILMhld0p5JWPgoYdgwgT4\n/nu7o/Fdf2jzB7af3K59715A7PqEjY6ONuvX61Shyn7GwBNPwBtvwLPPwssv2x2R70pJT6HmP2sS\nVT2KH+78AZGcvrJTRSEiG4wx0fnV09G7qkRLT7dmeXzjDRg/Hl7S63CKJCggiIk3TWTJ/iVsPLbR\n7nBKNE3uqkTbtw/mzLHO2N9+G/REs+jujrqboIAg/rPlP3aHUqJpclclUmKi1R3TsCFs3251xWhi\nd49KpSsxoOEAPtv2GSnpKXaHU2JpclclTmysdQ/Ut9+21sP1emq3e6DtAxy/eJyPf/3Y7lBKLE3u\nqsQwBv77X+jQAS5csJ6VZ/St35d2NdsxeeVkktOS7Q6nRNLkrkqE336D6Gi4/XZo1QrWr4dOneyO\nyn+JCC/0eIED5w7w1pq37A6nRNLkrkqE7dth40Z49VVragHtivG8fg36MbjxYF75+RVOXNRrG4ub\nJnfll9LSrDsnTZ5srffqBUeOwNNPQ2CgvbGVJK/1fo1LaZd4dsmzdodS4mhyV34lMRH+9S9o1Ahu\nuw1mzYJLl6yymjXtja0kalylMY92eJR///pvlh9cbnc4JYomd+U3vvrKSuDjx1s3s/7yS9i0CUqX\ntjuyku3vPf7ODRVu4IFvHiA1PdXucEoMTe7KJ6WlWX3njz12dS6YFi1gxAhYs8a6i9Itt+gdlLxB\nmVJlmNpvKjtP7WTyysl2h1NiaO+j8hnGWFeTLlwI334LZ85AcLB1tt6vH9SvDx/rsGqvNLjxYIY2\nHcqLy1/kzpZ3ElFRb7PsaTpxmPI6xkB8PGzdClu2WFeOPvWUVdawoZXUBw6EIUOgb1+dmtdXHDh3\ngMb/asyQxkOYO2Ku3eH4LFcnDtMzd2WrxETYuxeioqz1Rx+FmTPh3Lmrddq3v5rclyyxztR1xIvv\nqVOxDn/u9Gf+seIfxJ6MJbJqpN0h+TXtkVQeZQwkJEBGhrW+ahX89a9Wf3i9elC+vJW8U5xTkNSr\nB6NHw7Rp8PPPcPasdVekTDfcoIndl/25058JDgzmlZ9fsTsUv6d/JqpAUlLg5EnrzDohwUq+md0k\nlSpZX26+8w4cPw4nTliPy5fh4EErMf/0k3UhUePG0K4d3HsvtGx5df8TJth3bMrzwsqE8VjHx3h1\nxauMaz+OjuEd7Q7Jb2mfu5cxxkqgaWlXH6mpEBoK5cpZy3v3Ws9Zy+vUsborEhOtM96s5Wlp1jwq\nDRpYl+HPm2cl3EuXrJtBJyXBXXdZXSMbNlhn1klJcPHi1edPP4WuXa0vNEePvj7ulSuhc2f44gtr\nhsVq1eB3v7v6PHYsVKli7a9UKeuhSqYLKRdo9E4jwsuHs+YPa3CIdiAUhF/3ub/8spWEwEqGxkCt\nWta/8gB/+Qts23a1LCPDOlN85x2r/I9/hD17rpYbY80SOHWqVT58OBw6dG15167w5ptWeY8e1tlr\n1vKBA+H1163yFi2shJi1/Pbb4ZVXrJtD1K59beJNS7P6ml95xTobrlTp+mN+4QX429+s5Ny06fXl\nb75p7SM+HgYMuL78ww+t5H7okDUOPJPDAWXKwI03Wsk9Pd06Ey9TxkrMZcpYj4oVrfrt2sH06VaM\nFSpYz5UqWWflAEOHWo/clCmTe5kqGcoGlWVy78nc9dVdfLr5U8ZGjbU7JL/kk8n96FGIi7u6nn0e\n7rNnreQrYiUvESvZZsrIsBJuZrnDAUFBV8vLl7dukCxytU5Y2NXyunWhcuVryyOyjOzK7EPOLBex\nPlzAqjtggNVvHBhoncEGBkKXLlZ56dJWks8sz3y0a2eVh4VZl9VnfW1gIDRpYpXXqWP1UWfff7Vq\nVnnr1lZXSXCw1Vb2M+j27a/t486uXj3rzkVKFcXtLW/n7bVvM/HHiQxqPIjKpSvbHZLf0W4ZpZQt\nlu5fys3/uZnOtTuzdOxSvd+qi/Qeqkopr9ajbg/+2fef/HTwJ/6x4h92h+N3fLJbRinlHx5u/zCr\n41fz7I/PUi64HOPaj7M7JL+hyV0pZRuHOJh5y0ySUpMY/914ygWV0y9Y3US7ZZRStioVUIo5w+fQ\no04P7ltwH59s/sTukPyCJnellO2CA4P5YtQXdI3oyt1f3c3UtVOxa7CHv9DkrpTyChVDKvL1mK8Z\n3Hgwj3z/COMWjiMtI83usHyWJnellNcIDQpl/sj5PNHpCd5d/y6DPhtEUmqS3WH5JE3uSimvEuAI\n4PW+r/P+wPdZFLeInjN7En8+3u6wfI4md6WUV7q/7f18PuJzthzfQv2p9Xnwmwc5lHDI7rB8hiZ3\npZTXGhY5jG0PbePuVnfz71//TdNpTXl2ybMkJCfYHZrX0+SulPJq9SrV4/1B77Nn/B76NejH5JWT\nafBOA15b+RoXUy7mv4MSSpO7UsonRFSMYP7I+az74zqia0bz1OKnqPt2XV79+VVOXjxpd3heR5O7\nUsqntKnRhu9u/45V964iqnoUz/74LOFvhnPnl3ey+vBqHR/vpLNCKqV82o6TO3hv/XvM3DyT85fP\nE1U9itua38bwyOHUrVTX7vDcztVZITW5K6X8woWUC/x36395f8P7bDy2EYAmVZrQObwzbWu2pV3N\ndrSp0YYAR4DNkRaNW5O7iPQD3gYCgI+MMf/IVh4MfAK0BU4Do4wxB/LapyZ3pZSnxJ2J45vd3xCz\nN4ZfjvzC6UunAYioEEHXiK6MbDaShpUbUqdiHYIDg22OtmDcltxFJADYDfQB4oF1wBhjTGyWOg8B\nLY0xfxKR0cCtxphRee1Xk7tSqjgYYzh8/jDLDizjlZ9fYe/ZvVemNQh0BFKvUj3KBZUjNCiUGmVr\nEF4+nLJBZSlTqgylA0sTEhhyzaN0qdKElgol0BFIgCOAQEdgro8Aub7cIY4i3ZjEncm9EzDJGHOz\nc/0Z5w/s1Sx1FjnrrBaRQOA3oKrJY+ea3JVSdkhKTWLjsY0cOHeA2JOxxJ2J42LqRS6mXORI4hHi\nz8eTnJbs0RjeG/Aef4r+U6Fe684bZNcCDmdZjwc65FbHGJMmIglAGHAqW1D3A/c7Vy+IyC4X2rdD\nFbLF7if0uHyLPx6XPx4TFPC4Hpz0IA/yYGHbisi/imvJPaf/H7KfkbtSB2PMB8AHLrRpKxFZ78on\no6/R4/It/nhc/nhM4J3H5co493igdpb1cOBobnWc3TIVgDPuCFAppVTBuZLc1wENRaSuiAQBo4EF\n2eosADLvjTUc+DGv/nallFKelW+3jLMPfRywCGso5MfGmO0i8gKw3hizAPg38KmIxGGdsY/2ZNDF\nwOu7jgpJj8u3+ONx+eMxgRcel20XMSmllPIcnVtGKaX8kCZ3pZTyQ36V3EVkhIhsF5EMEYnOsr2O\niFwSkU3Ox/QsZW1FZKuIxInIVHFeOiYilUXkBxHZ43yu5NwuznpxIrJFRNpk2ddYZ/09IjI2vzaK\nelzOsmec+90lIjdn2d7PuS1ORJ7Osr2uiKx1xjjH+SU5IhLsXI9zltcpbBuFISKTRORIlt/R773h\nGO3gzp+rO4nIAef7eJOIrHdu8/jfSW5tFOE4PhaREyKyLcs2244jrzaKxBjjNw+gKdAYWAZEZ9le\nB9iWy2t+ATphjdX/Dujv3P4a8LRz+WlgsnP59856AnQE1jq3Vwb2OZ8rOZcr5dWGG44rEtgMBAN1\ngb1YX3oHOJfrAUHOOpHO18wFRjuXpwMPOpcfAqY7l0cDcwrbRiF/d5OAJ3LYbtsx2vQeduvP1c2x\nHQCqZNvm8b+T3NoownF0BdqQJSfYeRy5tVHk35fdbxgPvQmX4UJyB2oAO7OsjwHedy7vAmpkqbfL\nufw+1tw6ZK2X9bVZ6+XVhhuO6xngmSzri5xvqk7Aouz1nG+eU0Cgc/uVepmvdS4HOutJQdsowu9s\nEjknd9uO0ab3rlt/rm6O7QDXJ3eP/53k1kYRj6UO1yZ3244jtzaKeox+1S2Tj7oi8quI/CQiNzm3\n1cK6ACtTvHMbQDVjzDEA5/Pvsrwm+3QMtfLZnlsbRVXQWMKAc8aYtBxiuWYKCSBzComCtlEU45z/\nln6c5V9vO4/RDt4US3YGiBGRDWJNJQLF83eSWxvuZOdxeOR37sr0A15FRBYD1XMommiM+V8uLzsG\n3GCMOS0ibYGvRKQZLk6bkD2EXF5T0O3X7rRwx5XbvnP60M4vloLGn1sbucrrGIH3gBed+3gR+Cdw\nbyHad+cx2sGbYsmuizHmqIj8DvhBRHbmUdcjfyc2KI7j8Mix+1xyN8b0LsRrLgOXncsbRGQv0Ajr\nEzI8S9WsUyscF5EaxphjIlIDOOHcntt0DPFA92zbl+XTRpGOK49YyGX7KaCiiAQ6z1yz1s/cV7xc\nO4VEQdvIlavHKCIfAt9ki8vV9t19jMXNm2K5hjHmqPP5hIh8CbSneP5OcmvDnew8Do/8zktEt4yI\nVBVrXnpEpB7QENjn/NcoUUQ6Or/RvgvIPEvOOqXC2Gzb73J+w90RSHDuZxHQV0QqObsU+mL1nebV\nRlEtAEaLNQqkrvO4fiGXKSOM1aG3FGuKiJyOK6cpJArURmEPxPlmz3QrkDmSwc5jtINbf67uIiKh\nIlIucxnr/b2N4vk7ya0Nd7LzOHJro2jc/aWLnQ+spBCPdZZ+nKtfpA0DtmONPNgIDMrymmisN+le\n4F9cvWo3DFgC7HE+V3ZuF2Cas/5Wrv2C814gzvm4J782inpczrKJzv3uIssoHKxv4Hc7yyZm2V4P\nK3HFAZ8Dwc7tIc71OGd5vcK2Ucjf3afOn+cWrDd7DW84Rpvex277uboxpnrOv5/Nzr+licX1d5Jb\nG0U4ls+wumpTnX9X99l5HHm1UZSHTj+glFJ+qER0yyilVEmjyV0ppfyQJnellPJDmtyVUsoPaXJX\nSik/pMld+RwRCZOrs0f+JldnkzwnIrEeaK+7iHyTf81rXrNMss3g6dx+t4j8y33RKZUzTe7K5xhj\nThtjoowxUVizPr7pXI4CMvJ7vfPqVKX8miZ35W8CRORDsea/jxGR0nDlTPoVEfkJeMR51fJ8EVnn\nfHRx1uuW5b+CXzOvygTKisg8EdkpIrOcVx8iIr2c9baKNeFZcPaAROQeEdntbLtLMf0cVAmnyV35\nm4bANGNMM+Ac1tXJmSoaY7oZY/4JvI11xt/OWecjZ50ngIed/wncBFxybm8NPIo193s9oIuIhAAz\ngFHGmBZYczU9mDUY57QKf8dK6n2cr1fK4zS5K3+z3xizybm8AWve7kxzsiz3Bv4lIpuwpjso7zxL\nXwm8ISITsD4MMqcO/sUYE2+MyQA2Offb2NnebmedmVg3gsiqA7DMGHPSGJOSLQalPEb7HpW/uZxl\nOR0onWX9YpZlB9YNOS5xrX+IyLdY87usEZHMmSyz7zeQnKdqzYnO8aGKnZ65q5IqBhiXuSIiUc7n\n+saYrcaYycB6oEke+9gJ1BGRBs71O4GfstVZC3R3jvApBYxw1wEolRdN7qqkmgBEi3Xnp1jgT87t\nj4rINhHZjNXf/l1uOzDGJAP3AJ+LyFaskTrTs9U5hnULwdXAYqxZSZXyOJ0VUiml/JCeuSullB/S\n5K6UUn5Ik7tSSvkhTe5KKeWHNLkrpZQf0uSulFJ+SJO7Ukr5of8HvleeKXth8mIAAAAASUVORK5C\nYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f81cc7010b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):\n",
    "    plt.plot(thresholds, precisions[:-1], \"b--\", label=\"Precision\")\n",
    "    plt.plot(thresholds, recalls[:-1], \"g-\", label=\"Recall\")\n",
    "    plt.xlabel(\"Threshold\")\n",
    "    plt.legend(loc=\"center left\")\n",
    "    plt.ylim([0, 1])\n",
    "plot_precision_recall_vs_threshold(precisions, recalls, thresholds)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.79004370131103929"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred_90 = (y_scores > 70000)\n",
    "precision_score(y_train_5, y_train_pred_90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.76701715550636418"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_5, y_train_pred_90)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### 3.3.5 ROC曲線"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "* 偽陽性率(FPR)に対する真陽性率(TPR、 再現率のもうひとつの名前)をプロットしたもの\n",
    "* FPR=1-真陰性率(TNR、特異度)\n",
    "* つまり、ROC曲線は1− 特異度に対する感度をプロットした曲線"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "fpr, tpr, thresholds = roc_curve(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEKCAYAAAAMzhLIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl0FFX2wPHv7c5G2BcRZVdhZEeMLCIggsimOKJsyiqg\nKKOjoujoGWccfzo6yjjOuICAOo4O7gqKyqgogrLKvsoiISh7CEv27vv7o5oQY5ZOSKe6k/s5J8eu\n7uqqS5nUrfde1X2iqhhjjDEF8bgdgDHGmPBmicIYY0yhLFEYY4wplCUKY4wxhbJEYYwxplCWKIwx\nxhQqZIlCRGaLyAER2VDA5yIiz4rIdhFZJyIdQhWLMcaYkgtli+IVoG8hn/cDmgV+JgIvhDAWY4wx\nJRSyRKGqi4AjhawyCPi3OpYCNUTknFDFY4wxpmSiXNx3fWBPruWkwHs/511RRCbitDqoXLnyxRde\neGGZBGiMMflRBZ8qfr+iOe8pPr+zpIF1FCUrW/F4nDc18JnPp/hU8cjp7fkD3/cr+PzO56rOcpbP\nX+JYs1MO4M84CX7fIVU9qyTbcDNRSD7v5VtPRFVnADMAEhISdOXKlaGMyxhTxvx+JdPnJ9Pn5/CJ\nTLJ8fudk6ddcJ1Dl4PFM9h5NIybKg8/nJ9uvHDieQUaWj9hoL1k+Pxv3HqNe9Th8gW1+vzuZ88+q\ngl818EPOCdivmnOSVoWt+49TIz6aStHenJP2qe+diic101fsf1/eb3goWXdOXLSHGpViiPIK0V4P\nPx4+SaemtaheKRpV6NC4Jl4RPAJej+D1evh4ziv40lJ4Z8bfd5dgl4C7iSIJaJhruQHwk0uxGGNw\nrmSPp2dx4HgG+1LS8amS7VOyfX72Hk2jUowXv1/J8jknzW37j1O7Sixb9h2jcmwUHhH8gRPqqSvu\nlbuTaVa3Cll+JSvbz6afj3FO9TiyfEq238/R1KyQ/7sOnyysF/yXjqZmcZSiY6oSG8WJjGya1I7H\n6xGiPM6pPyk5lfaNauD1eIjyCD6/cuRkJhfWq+qcvD1ClEdIScuieqVoaleJJcrrvFc5Nopa8THE\nRnuoGR9DfEwU0YGkULtKDJWivYjkd4192t69e5k0aRJDhw7lxhtvZGTnhwGQGX8P+hjk5WaimAtM\nFpE5QCcgRVV/1e1kTEWRlukj2+8n26dk+fwcOJ6Bz69kB0682T4/SUfT2JeSTkyUh637nKtff2Cd\nLJ+ftXtSaFQ7nmyfnxMZ2Wz5+TiN68Tj8ztX7adO3pk+P0nJadSMj87ZfkmulIO1cnfyL5Z/Tkn/\n1TpRHiEu2ktctJca8dHOlbFH8HrIea0Kx9KyqB4fTZv61YnyeIjyCimpWZxVNZYa8dHERHk4lpZF\n49qVifZ6iPYK2X6lVuUYPAIigidw1S0IIjjLHue/AjlJ79R73lPfCSzHxxR9wi5rqsrMmTOZMmUK\nWVlZDBgwoNS2HbJEISL/BS4H6ohIEvAwEA2gqi8C84H+wHYgFRgbqliMKaksn5/UTB/H07PYcySN\njGwfvsAV9d6jafj9yvYDJ6gU4wX4RTeJ3w9rk47SuHY8WT5lzZ6j1K0a+4suDicBKGlZpXeS3rr/\n+C+WN+w9VuC6yQVczUd5hJqVY8j2+enQqKZzxev1sPvwSdo3rOGcoD2C1yscPJ5Bq3OrcyI9m0a1\nKxHl8eD1OCdW578gAjXjY4j2Op/Fx3ipFO0lyuuc6CvHROH1hNeJN5Ls2LGDCRMmsHDhQnr27MlL\nL73E+eefX2rbD1miUNXhRXyuwO2h2r+pWHL3cZ9Izw6czJ1+7pS0rJwr86zAlfTeo2nERXlZm+Sc\nvJNTM9l9OBWPOF0Ce4+mlVpsW/adPnEfOZmZ8zq/Lpf4GC8xUc5JOCPbT63KMdSqHOOclD2CICSn\nZtK4djzNz67KoRMZ/Obsqni9HqI9ztVxtl+pVy2OKK8HCWwzLtqb0+2R+wReOTaKaK8nZ/tRHich\nmMiyfv16Vq1axYwZMxg/fnypt3bc7HoyBoD0LB8Hj2ew/1h6zoBhll/ZfyydjCwfSUfT2Lj3GHWq\nxLBl33FiojwcPpHJoRMZZGSX/G6Q4qhTJYZDJzKpFhdFQpNaeD1CtFdIPplF24bVOXoyi2ZnVzl9\nQg+cjL0ipGZm07BWPNGBE/BZVWOJifJQKdpL5ZgooqOc/u1or4Rdd4YJXxs2bOD7779n1KhRXHvt\ntezcuZPatWuHZF+WKEyxqDp92amZPk5mZHM8PZvjGVkkn3SujvckpxLt9TgDnn4/2w+cIDPbT0pa\nFnHRXvYfS+fg8Qxqxsew71j6L66wz1RMoD/6ZKaP+jUqEe09fQW9+3AqHRrXyOn6SEnL4uyqcbSu\nX43UTB/nn1UFr0doWCueqnFRRHmE2pVjiY/15pzgjQkHmZmZPPbYYzz22GOcffbZDBkyhLi4uJAl\nCbBEUaGlZmZz+EQmh09mkp7lI/FIKj6/kpScys9H00nN9HE0LZM9R9I4mZlNaqaPzFK6gs9vMDMu\n2kOzulWpEhuVcxdIUnIaHRrVpG61WGKjPJx3VhV8fuXcGnHERnk5q2osVeOigrobxJhIt2zZMm6+\n+WY2btzITTfdxN///nfi4uJCvl9LFOXU/mPp7DmSysHjGaSkZfF9YjI+P2Rk+9iwN4XDJzM5np59\nRvs4t3ocP6Wk065hDY6nOXednFM9jqTkNNo0qE5MYKAyOTWL8+pUpk6VWOd+b5RalWOpHOOlWqVo\n6laNtZO8MUXYu3cv3bp14+yzz+ajjz4q1buaimKJIkKpKj8eTmXjTynsS0lndeJRKsd6+fFwKlt+\nPsaxYiSB8+pU5qeUNC5pUgufX2lWtwqpmT7qVI2lUa14asbHcGG9qtSrHpfTdWOMKRvbtm2jefPm\n1K9fnzfffJNevXpRrVq1Mo3BEkUY8vuVLfuOk5Scyq5DJ9l7NI11SSmoKpk+ZfPPBd/ueEpMlIfM\nbD9N61Tm7Gqx1IyPoWpcFF0vqEOM10PtKrG0bVCduGhvGfyLjDHFdfToUe677z5mzpzJV199Rffu\n3fntb3/rSiyWKFy2/1g6K348wurEo2z8KYWlO4N/gvSULufVpmPTWqRn+bioUQ2an12VRrXi7TZH\nYyLU3LlzmTRpEvv27ePee+/lkksucTUeSxRl6Hh6Ft/tOMzi7Yf4ZMM+Dh7PKHT9mvHR1K0aR6Pa\n8XRqWosL61WjRnw0sVGewFOoMWUUuTGmrIwfP55Zs2bRpk0bPvzwQxISEtwOyRJFqGRm+9my7xgb\nfzrGF5v38/nmA4Wuf271ODo2rUVCk1okNKlJs7pVbSzAmArCef7YKS+SkJBA48aNmTp1KjEx4XEx\naImilGT5/CzccoCP1v3Mwq0HCr2jqGZ8NJ3Pq82g9ufSu8XZ1kVkTAW2Z88ebr31VoYNG8bIkSO5\n9dZb3Q7pVyxRnIFVu48we8mP7E1OY82eo/muU6dKLDckNKBDo5p0b16H2CgbPDbGgN/vZ/r06Uyd\nOhWfz+faQHUwLFEU04+HTvLa0t38d3livtU2O59Xiz4t63Fly7NpWCvehQiNMeHuhx9+YPz48Sxa\ntIjevXszY8YMmjZt6nZYBbJEEQRV5fVliTw+fzMn8ySHyy6oQ+VYL4M7NODy39QlJsq6kYwxhdu0\naRPr1q1j9uzZjBkzJuwfOLVEUYgNe1OYvmgn89b+cj6lOlViGXdZE0Z0bGR3HhljgrJ27VrWrFnD\n6NGjGTRoEDt37qRmzZpuhxUUSxT5eGvlHu57Z92v3h/U/lx+d0UzLqhbxYWojDGRKCMjg0cffZS/\n/vWvnHPOOQwdOpS4uLiISRJgieIXVv54hClvr+XHw6k5751TPY6nh7SjY5NadneSMaZYvvvuO26+\n+WY2b97MqFGjmDZtWpkU8SttliiAxMOpPPvlD7yzKinnvYsb1+TP17Sidf3qLkZmjIlUe/fupUeP\nHtSrV4/58+fTr18/t0MqsQqfKN5ZlcSUt9fmLNevUYn5d3ajeqVoF6MyxkSqzZs306JFC+rXr89b\nb71Fr169qFq1qtthnZEK25fi9yvjX135iyTx1A3tWDy1pyUJY0yxJScnM27cOFq2bMk333wDwLXX\nXhvxSQIqcIvij3M38Pnm/YBTVG/WmATiYyrs4TDGnIH333+f2267jYMHD/LAAw+4XsSvtFXIM+O9\nb6/l7cB4xLQh7biuQwOXIzLGRKpx48bx8ssv0759ez7++GM6dOjgdkilrsIliteX7c5JEh2b1rIk\nYYwpttxF/Dp37kyzZs2YMmUK0dHls9u6QiWKT9b/zIPvbwDgniub87tezVyOyBgTaXbv3s0tt9zC\niBEjGDVqFBMnTnQ7pJCrMIPZy3cdYdLr3wOBB+csSRhjisHv9/Pcc8/RunVrFi9eTFZWltshlZkK\n0aLIyPYx6T+rALigbhX+PqS9yxEZYyLJ1q1bGT9+PIsXL6ZPnz5Mnz6dJk2auB1WmSn3iSL5ZCZX\n/n0Rh09m4vUI7992KR6bEMgYUwxbt25l48aNvPLKK4waNSrsi/iVtnKfKG75zyoOnXCmHJ0zsTNV\n48rnYJMxpnStXr2aNWvWMHbsWK655hp27txJjRo13A7LFeV6jGLD3hSW7zoCwOwxCVzSpJbLERlj\nwl16ejp/+MMfuOSSS/jTn/5Eeno6QIVNElCOE4XfrzlPXV/T7lyuuPBslyMyxoS7JUuW0L59ex5/\n/HFGjRrFmjVrIrKIX2krt11PT/9vK1v2HSc2ysMjg1q5HY4xJszt3buXnj17Ur9+fT777DP69Onj\ndkhho1y2KHx+5bmFOwD4fe/mNrmQMaZAmzZtAqB+/fq8++67rF+/3pJEHuUyUZwalwCY2P08FyMx\nxoSrI0eOMGbMGFq1asWiRYsAuPrqq6lSxSYmy6tcdj1988NBALo3Pwuv3QprjMnj3Xff5fbbb+fw\n4cM8+OCDdOzY0e2Qwlq5TBRfbjkAwHUX1Xc5EmNMuBkzZgyvvvoqHTp04NNPP6V9e3sAtyjlLlFs\n23+cLfuOA9Cj+VkuR2OMCQe5i/hdeumltGjRgnvuuYeoqHJ3CgyJkI5RiEhfEdkqIttF5P58Pm8k\nIgtFZLWIrBOR/me6z/8uTwSgX+t61Kxsg9jGVHS7du2iT58+/Pvf/wZg4sSJTJ061ZJEMYQsUYiI\nF3gO6Ae0BIaLSMs8qz0EvKWqFwHDgOfPZJ+qysfrfgZgfDcbxDamIvP5fDz77LO0bt2apUuX5rQq\nTPGFskXREdiuqjtVNROYAwzKs44C1QKvqwM/nckO1yalcOB4BrUqx3BRw4r7FKUxFd3mzZvp1q0b\nd955Jz169GDjxo2MGTPG7bAiVijbXvWBPbmWk4BOedb5E7BARH4HVAZ657chEZkITARo1KhRgTt8\neckuADo1rWWF/4ypwLZv387WrVt57bXXuPHGGytcEb/SFsoWRX7/Z/K2/YYDr6hqA6A/8JqI/Com\nVZ2hqgmqmnDWWQUPUKekOfXhL6hr90EbU9GsWrWK2bNnA87zELt27eKmm26yJFEKQpkokoCGuZYb\n8OuupZuBtwBU9TsgDqhTkp1lZPv4aqvz/MSwjgW3Oowx5UtaWhr3338/nTp14i9/+UtOEb9q1aoV\n8U0TrFAmihVAMxFpKiIxOIPVc/Oskwj0AhCRFjiJ4mBJdrb4h0MA1K4cQ/0alUoaszEmgixatIh2\n7drxxBNPMGbMGFavXm1F/EIgZGMUqpotIpOBzwAvMFtVN4rII8BKVZ0L3AO8JCJ34XRLjdES3prw\n8XrnbqcGNS1JGFMR7N27l169etGwYUM+//xzevXq5XZI5VZIbyRW1fnA/Dzv/THX601A19LY17qk\nFAAmXX5BaWzOGBOm1q9fT5s2bahfvz7vv/8+PXv2pHLlym6HVa6Vi6KAR05msv3ACQC6XlDb5WiM\nMaFw6NAhRo4cSdu2bXOK+A0cONCSRBkoF48mzlvrjJF3bFLLpjo1ppxRVd5++20mT55McnIyDz/8\nMJ065b3T3oRSuUgUm38+BkCr+naXgzHlzejRo3nttddISEjgiy++oE2bNm6HVOGUi0QxZ4XzXN/A\ntue6HIkxpjTkLuLXo0cP2rZty+9//3urz+SSiB+jOHg8I+d1uwbVXYzEGFMadu7cSe/evXnllVcA\nuPnmm5kyZYolCRdFfKI4NYgtAlHeiP/nGFNh+Xw+nnnmGdq0acOKFSvweOzvOVxEfIpOPHISgN/a\nJEXGRKxNmzYxbtw4li1bxoABA3jxxRdp0KCB22GZgHKQKFIBOLe6PWhnTKTatWsXO3bs4I033mDY\nsGFWnynMRHyieO/7vQC0tfEJYyLKihUrWLNmDRMmTGDAgAHs3LmTqlWruh2WyUfEdwIeC1SMbV3f\nEoUxkSA1NZUpU6bQuXNnHn/88ZwifpYkwldEJ4qU1CxOZvoAOKe6FQIzJtx99dVXtG3blqeffpoJ\nEyZYEb8IEdFdT/uPO1cijWvHW5+mMWEuKSmJK6+8ksaNG/Pll1/Ss2dPt0MyQYroFsWPh5w7nhrW\njHc5EmNMQdauXQtAgwYN+PDDD1m3bp0liQgT0Ynih8AzFM3Ptr5NY8LNwYMHGTFiBO3bt+frr78G\noH///sTH24VdpInorqfdh50WRdOzrHqkMeFCVZkzZw533HEHKSkp/PnPf6ZLly5uh2XOQFCJIjBD\nXSNV3R7ieIpl92HnGYrGtewKxZhwMXLkSF5//XU6derErFmzaNWqldshmTNUZNeTiAwA1gP/Cyy3\nF5H3Qx1YMHYGxijq26x2xrjK7/fnFPLr2bMn06ZNY8mSJZYkyolgxigeAToBRwFUdQ3g+jRyJzKy\ncwoCNrIWhTGu2b59O7169eLll18GnCJ+d911F16v1+XITGkJJlFkqerRPO+VaF7r0rQvJT3ndbQV\nAzSmzGVnZ/PUU0/Rpk0bVq9eTUxMjNshmRAJZoxis4gMATwi0hS4E1ga2rCKtuOgc8dTt2Z1XI7E\nmIpnw4YNjB07lpUrVzJo0CCef/55zj3X5oMpr4K5FJ8MXAz4gfeAdJxk4aqfj6YB0MDGJ4wpc4mJ\niezevZs5c+bw/vvvW5Io54JpUVylqlOBqafeEJHrcJKGazb+5Ex/WqdKrJthGFNhLFu2jLVr1zJx\n4kT69+/Pzp07qVKlitthmTIQTIvioXzee7C0Aymu/YGB7LrVrE6MMaF08uRJ7r77brp06cKTTz5J\nRobzt2dJouIosEUhIlcBfYH6IjIt10fVcLqhXOUJlHaqFW8DaMaEypdffsmECRPYuXMnkyZN4q9/\n/SuxsdaKr2gK63o6AGzAGZPYmOv948D9oQwqGCmB8uJ1q9kvrTGhkJSUxFVXXUXTpk35+uuv6d69\nu9shGZcUmChUdTWwWkReV9X0gtZzy+pE547ds2yMwphStXr1ai666CIaNGjAvHnz6NGjB5Uq2U0j\nFVkwYxT1RWSOiKwTkW2nfkIeWSFUlZgoJ/TaVazryZjSsH//foYOHUqHDh1yivj17dvXkoQJKlG8\nArwMCNAPeAuYE8KYinQsPZvMbD9x0R6qxEZ0XUNjXKeq/Oc//6Fly5Z88MEHPProo1x66aVuh2XC\nSDCJIl5VPwNQ1R2q+hDgajH5kxnZAFSvFG0TFhlzhkaMGMHIkSP5zW9+w5o1a3jwwQeJjo52OywT\nRoK5HM8Q52y8Q0RuBfYCdUMbVuFODWRXttaEMSXi9/sREUSEPn360KVLF26//Xarz2TyFUyL4i6g\nCnAH0BWYAIwLZVBF+TnFeSq7dmUbnzCmuLZt20bPnj2ZPXs2AGPHjuWOO+6wJGEKVOQluaouC7w8\nDowEEJEGoQyqKJnZzmMcp6rHGmOKlp2dzbRp03j44YeJi4uzQWoTtEJbFCJyiYhcKyJ1AsutROTf\nuFwUMDnV6XpKaFLLzTCMiRjr1q2jc+fOTJ06lX79+rFp0yZGjBjhdlgmQhSYKETkceB14EbgUxF5\nEFgIrAWal014+Vvx4xEAzqlu5TuMCUZSUhJ79uzh7bff5t133+Wcc85xOyQTQQrrehoEtFPVNBGp\nBfwUWN4a7MZFpC/wD8ALzFTVv+azzhDgTzhzXKxV1SIvcw6dyAQgy+f6tBjGhK1vv/2WdevWceut\nt+YU8atc2eaXN8VXWNdTuqqmAajqEWBLMZOEF3gO59mLlsBwEWmZZ51mwANAV1VtBfw+mG1HBQo9\nNaxlfazG5HXixAnuvPNOLrvsMp5++umcIn6WJExJFdaiOE9ETpUSF6BJrmVU9boitt0R2K6qOwFE\nZA5OK2VTrnUmAM+panJgmweCCfpEuvMcReNa9otvTG4LFixg4sSJJCYmcvvtt/PYY49ZET9zxgpL\nFIPzLP+rmNuuD+zJtZyEM/d2bs0BRGQJTvfUn1T107wbEpGJwESARo0aUfm4U3qqTlW7PdaYU/bs\n2cOAAQM4//zzWbRoEZdddpnbIZlyorCigF+c4bbze2Q676BCFNAMuBxoAHwjIq3zztGtqjOAGQAJ\nCQm6+3AqgJXvMAZYtWoVF198MQ0bNmT+/Pl069aNuDi70cOUnmAeuCupJKBhruUGOAPiedf5UFWz\nVHUXsBUncQSlhs1FYSqwffv2ccMNN5CQkJBTxO/KK6+0JGFKXSgTxQqgmYg0FZEYYBgwN886HxCo\nGxV4VqM5sLOwjWquNknlGHuS1FQ8qsqrr75Ky5YtmTdvHo899pgV8TMhFXTfjYjEqmrQj0KraraI\nTAY+wxl/mK2qG0XkEWClqs4NfNZHRDYBPuBeVT1c2HZ9uTKFFQQ0FdGwYcN466236Nq1KzNnzuTC\nCy90OyRTzhWZKESkIzALqA40EpF2wHhV/V1R31XV+cD8PO/9MddrBe4O/AQlO/DsRIOadmusqThy\nF/Hr378/3bp147bbbsPjCWWngDGOYH7LngUGAocBVHUtLpYZ10CLYv+xsJt0z5iQ2LJlC927d2fW\nrFkAjB49msmTJ1uSMGUmmN80j6ruzvOeLxTBBCPb7ySKixrWdCsEY8pEVlYWjz32GO3atWPTpk1U\nqVLF7ZBMBRXMGMWeQPeTBp62/h3g2lSop1oU+6xFYcqxNWvWMHbsWNasWcP111/PP//5T+rVq+d2\nWKaCCiZRTMLpfmoE7Ac+D7znilND2c3q2tWVKb/27dvHvn37ePfdd7nuuqKKIBgTWsEkimxVHRby\nSIKUme1HgOrxNlWjKV8WL17MunXruO222+jbty87duwgPj7e7bCMCWqMYoWIzBeR0SJSNeQRFcET\nuCX256PW9WTKh+PHjzN58mS6devGM888k1PEz5KECRdFJgpVPR94FLgYWC8iH4iIay0Mf2CMok2D\n6m6FYEyp+eyzz2jdujXPP/88d955J99//70V8TNhJ6j761T1W1W9A+gAHMOZ0MgV6dnODVexUXZr\noIlse/bsYeDAgcTHx7N48WKeeeYZu7PJhKUiz7YiUkVEbhSRecBy4CDgWr2AqMC94ylpWW6FYEyJ\nqSrLly8HoGHDhnzyySesXr3aSnCYsBbMZfkGoDPwpKpeoKr3qOqyEMdVoLRMp0XR/GzXh0uMKZaf\nf/6ZwYMH06lTp5wifr1797YifibsBXPX03mq6g95JEGK9jqD2amZ2S5HYkxwVJVXXnmFu+++m/T0\ndJ544gm6du3qdljGBK3ARCEiT6vqPcC7IvKryamDmOEuJE4FUr+G3RFiIsOQIUN455136NatGzNn\nzqR58+Zuh2RMsRTWongz8N/izmwXUqfueoqxwWwTxnw+HyKCx+Ph6quv5oorruCWW26x+kwmIhX4\nW6uqywMvW6jqF7l/gBZlE96vnRqjsERhwtXmzZvp1q1bThG/UaNGMWnSJEsSJmIF85s7Lp/3bi7t\nQIIV7XVCVv1Vb5gxrsrKyuLRRx+lffv2bN26lerV7VkfUz4UNkYxFGdWuqYi8l6uj6oCR/P/Vuid\nSg91qthDSSZ8rF69mjFjxrBu3TqGDh3Ks88+S926dd0Oy5hSUdgYxXKcOSgaAM/lev84sDqUQRXm\nVEsiLtqa8SZ87N+/n0OHDvHBBx8waNAgt8MxplQVmChUdRewC6dabNgITEdBbJTNl23ctWjRItav\nX8/tt99O37592b59O5Uq2cyLpvwp8LJcRL4O/DdZRI7k+kkWkSNlF+IvnWpRxFqLwrjk2LFj3Hbb\nbfTo0YNnn302p4ifJQlTXhV2tj013Wkd4KxcP6eWXXFqDDvGa4nClL358+fTqlUrpk+fzt13321F\n/EyFUNjtsaeexm4IeFXVB3QBbgEql0Fs+ccV+G+0JQpTxvbs2cOgQYOoXr063377LU8//TSVK7v2\np2BMmQnmbPsBzjSo5wP/xnmG4o2QRlWIUw/cWaIwZUFVWbp0KeAU8VuwYAHff/89nTp1cjkyY8pO\nMGdbv6pmAdcBz6jq74D6oQ2raKdqPhkTKj/99BPXXnstXbp0ySni17NnT2JiYlyOzJiyFUyiyBaR\nG4CRwEeB91ydh7RaXBQilihMaKgqM2fOpGXLlixYsICnnnrKiviZCi2Y6rHjgNtwyozvFJGmwH9D\nG1bhjqVb5VgTOtdffz3vvfcePXr0YObMmVxwwQVuh2SMq4pMFKq6QUTuAC4QkQuB7ar6f6EPrWBV\n44LJb8YEL3cRv2uvvZY+ffowYcIEq89kDMHNcNcN2A7MAmYD20TE1XZ43ap2O6IpPRs2bKBr1645\nRfxGjhxplV6NySWYv4S/A/1VtauqXgoMAP4R2rAKZ3c8mdKQmZnJn//8Zzp06MCOHTuoWbOm2yEZ\nE5aC6cOJUdVNpxZUdbOIuHrbhyUKc6ZWrVrFmDFj2LBhAyNGjOCZZ57hrLNce47UmLAWTKL4XkSm\nA68Flm/ExaKAANsPnHBz96YcOHz4MEePHmXevHkMHDjQ7XCMCWvBJIpbgTuA+wABFgH/DGVQRWld\nv5qbuzcRauHChaxfv5477riDPn368MMPPxAXF+d2WMaEvUL7cESkDdAXeF9Vr1HVq1X1b6qaXjbh\n5c9mtzPrEGUiAAAWwElEQVTFkZKSwi233MIVV1zBCy+8kFPEz5KEMcEprHrsH3DKd9wI/E9E8pvp\nzhVeuxvFBGnevHm0bNmSmTNnMmXKFFatWmVF/IwppsK6nm4E2qrqSRE5C5iPc3us66I89lS2Kdqe\nPXsYPHgwF154IR988AGXXHKJ2yEZE5EKuzTPUNWTAKp6sIh1y5QlClMQVeXbb78FThfxW7lypSUJ\nY85AYSf/80TkvcDP+8D5uZbfK+R7OUSkr4hsFZHtInJ/IetdLyIqIglBBW11nkw+kpKSuOaaa+ja\ntWtOEb/LL7/civgZc4YK63oanGf5X8XZsIh4cebavhJIAlaIyNzcz2QE1quKc1fVsmC3/ePhk8UJ\nxZRzfr+fl156iXvvvZfs7GymTZvGZZdd5nZYxpQbhc2Z/cUZbrsjTl2onQAiMgcYBGzKs95fgCeB\nKcFuuOU5dnusOW3w4MF88MEHXHHFFbz00kucd955bodkTLkSynGH+sCeXMtJ5JnHQkQuAhqq6kcU\nQkQmishKEVkJ4LExigovOzsbv9+ZhHHw4MG89NJLfP7555YkjAmBUCaK/M7mmvOhiAenjtQ9RW1I\nVWeoaoKqJgB4bYyiQlu3bh1dunThpZdeAuCmm25i/PjxNkeJMSESdKIQkeLefJ6EM9/2KQ2An3It\nVwVaA1+JyI9AZ2BuMAPa1qKomDIyMnj44Ye5+OKL2b17t9VmMqaMBFNmvKOIrAd+CCy3E5FgSnis\nAJqJSNNAEcFhwNxTH6pqiqrWUdUmqtoEWApco6ori9qw3R5b8axYsYIOHTrwyCOPMHz4cDZv3sx1\n113ndljGVAjB1Hp6FhiI85Q2qrpWRHoW9SVVzRaRycBngBeYraobReQRYKWqzi18CwXzWqKocJKT\nkzlx4gTz58+nX79+bodjTIUSTKLwqOruPP2/vmA2rqrzcZ7ozv3eHwtY9/Jgtgn2HEVF8eWXX7J+\n/XruvPNO+vTpw7Zt26z8hjEuCGaMYo+IdARURLwi8ntgW4jjKtRPR9Pc3L0JsaNHjzJhwgR69erF\n9OnTc4r4WZIwxh3BJIpJwN1AI2A/zqDzpFAGVZRGtePd3L0JoQ8//JCWLVsye/Zs7rvvPiviZ0wY\nKLLrSVUP4AxEh40Ym+GuXEpMTOSGG26gRYsWzJ07l4SEoCq6GGNCrMhEISIvkev5h1NUdWJIIgqC\n3R5bfqgqixcvplu3bjRq1IjPP/+czp07W30mY8JIMJfmnwNfBH6WAHWBjFAGVRR74K58SExMZMCA\nAXTv3j2niF/37t0tSRgTZoLpenoz97KIvAb8L2QRBcF6niKb3+/nxRdfZOrUqagqzz77rBXxMyaM\nBXN7bF5NgcalHUhxWNdTZLvuuuv48MMPufLKK5kxYwZNmjRxOyRjTCGCGaNI5vQYhQc4AhQ4t0RZ\nsK6nyJOdnY3H48Hj8TB06FAGDRrEmDFjrD6TMRGg0EQhzl9xO2Bv4C2/qv5qYLus2ZPZkWXt2rWM\nGzeOCRMmcOuttzJ8+HC3QzLGFEOhvf2BpPC+qvoCP64nCbAnsyNFeno6Dz30EAkJCSQlJVGvXj23\nQzLGlEAwYxTLRaSDqn4f8miCdPCEqzddmSAsX76c0aNHs2XLFkaPHs20adOoVauW22EZY0qgwEQh\nIlGqmg1cBkwQkR3ASZx5JlRVO5RRjL/SoGYlt3ZtgnTs2DHS0tL49NNPueqqq9wOxxhzBgprUSwH\nOgDXllEsQbMy4+FpwYIFbNy4kbvuuovevXuzdetWK79hTDlQ2BiFAKjqjvx+yii+fNkYRXhJTk5m\n7NixXHXVVcyaNcuK+BlTzhTWojhLRO4u6ENVnRaCeIIS5bVEES7ee+89br/9dg4ePMgDDzzAH//4\nR0sQxpQzhSUKL1CF/Oe+dpXXY49mh4PExESGDRtG69atmT9/PhdddJHbIRljQqCwRPGzqj5SZpEU\ng41RuEdVWbRoET169KBRo0Z8+eWXdOrUiejoaLdDM8aESJFjFOEoPJ7mqHh2795Nv379uPzyy3OK\n+F122WWWJIwp5wpLFL3KLIpiyvQFNROrKSV+v59//etftGrVisWLF/PPf/6Tbt26uR2WMaaMFNj1\npKpHyjKQ4qgaa1ewZenaa69l3rx5XHXVVUyfPp3GjV2tCWmMKWMlqR7rOrs7NvSysrLwer14PB6G\nDx/O9ddfz8iRI62InzEVUETePmTnqtD6/vvv6dixIy+++CIAw4cPZ9SoUZYkjKmgIjNRhO84e0RL\nS0vjgQceoGPHjuzbt4+GDRu6HZIxJgxEZNeT5YnSt3TpUkaPHs22bdsYN24cTz31FDVr1nQ7LGNM\nGIjIRGF5ovSdPHmSrKws/ve//9G7d2+3wzHGhJHITBTWV14qPv30UzZu3Mg999xDr1692LJlCzEx\nMW6HZYwJMxE6RmHOxOHDhxk9ejT9+vXj1VdfJTMzE8CShDEmX5GZKCxTlIiq8s4779CyZUveeOMN\nHnroIVasWGEJwhhTqMjserI2RYkkJiYyYsQI2rZty4IFC2jXrp3bIRljIoC1KMo5VeXLL78EoHHj\nxnz11VcsXbrUkoQxJmiRmSjcDiBC7Nq1iz59+tCrV6+cIn6XXnopUVER2ZA0xrgkIhOFZYrC+Xw+\n/vGPf9C6dWuWLVvGCy+8YEX8jDElFpGXljZGUbhBgwbx8ccf079/f1588UV7wtoYc0YiM1FYnviV\n3EX8Ro4cyfDhwxkxYoQ9c2KMOWMh7XoSkb4islVEtovI/fl8freIbBKRdSLyhYgEVb/aTn2/tHLl\nShISEnjhhRcAGDp0KDfeeKMlCWNMqQhZohARL/Ac0A9oCQwXkZZ5VlsNJKhqW+Ad4Mkgt12aoUas\ntLQ0pk6dSqdOnTh48KDNE2GMCYlQtig6AttVdaeqZgJzgEG5V1DVhaqaGlhcCjQIZsOWJ+C7776j\nXbt2PPnkk4wbN45NmzYxcOBAt8MyxpRDoRyjqA/sybWcBHQqZP2bgU/y+0BEJgITAWLqXWBdTzit\nCb/fz+eff06vXmE7a60xphwIZaLI73yu+a4ochOQAPTI73NVnQHMAIg9p5lW1BbF/Pnz2bhxI/fe\ney9XXHEFmzdvJjrapoU1xoRWKLuekoDc92U2AH7Ku5KI9AYeBK5R1YzgNl2xMsWhQ4e46aabGDBg\nAK+//npOET9LEsaYshDKRLECaCYiTUUkBhgGzM29gohcBEzHSRIHgt1wRWlRqCpz5syhRYsWvPXW\nWzz88MMsX77civgZY8pUyLqeVDVbRCYDnwFeYLaqbhSRR4CVqjoX+BtQBXg7cCdToqpeU9S2K0ie\nIDExkdGjR9OuXTtmzZpFmzZt3A7JGFMBiWq+wwZhK/acZrps+QraN6zhdighoap88cUXObPMLV26\nlEsuuQSv1+tyZMaYSCYiq1Q1oSTfjchaT+W1RbFjxw569erFlVdemVPEr3PnzpYkjDGuisxEUc4y\nhc/nY9q0abRp04ZVq1Yxffp0K+JnjAkbEVnrKcsXWd1lRbn66qv55JNPGDhwIC+88AINGgT13KEx\nxpSJiEwUcdER2RD6hczMTKKiovB4PIwZM4aRI0cybNgwK09ijAk7EXnGjfQy48uXL+fiiy/m+eef\nB2DIkCEMHz7ckoQxJixFZqKI0PNpamoq99xzD126dCE5OZnzzz/f7ZCMMaZIEdn1FImJYvHixYwe\nPZqdO3dyyy238MQTT1C9enW3wzLGmCJFZqKIwK6nUxMLLVy4kMsvv9ztcIwxJmiRmSgiJE/MmzeP\nzZs3c99999GzZ082bdpEVFREHnJjTAUWmWMUbgdQhIMHDzJixAiuueYa/vvf/+YU8bMkYYyJRBGZ\nKMKVqvLGG2/QokUL3nnnHR555BGWLVtmRfyMMREtIi9xw7XrKTExkbFjx3LRRRcxa9YsWrVq5XZI\nxhhzxiK0RRE+mcLv9/PZZ58B0LhxY7755huWLFliScIYU25EZKIIlxbFDz/8wBVXXEHfvn1ZtGgR\nAB07drQifsaYciUyE4XL+8/OzuZvf/sbbdu2Zc2aNcyaNcuK+Bljyq0IHaNwN1UMHDiQzz77jEGD\nBvH8889z7rnnuhqPMcaEUmQmChf2mZGRQXR0NB6Ph/HjxzNu3DhuuOEG15OWMcaEWmR2PZXxuXnp\n0qV06NCB5557DoDrr7+eIUOGWJIwxlQIkZkoyqhNcfLkSe666y4uvfRSjh8/TrNmzcpkv8YYE04i\ns+upDPLEN998w+jRo9m1axe33XYbjz/+ONWqVQv9jo0xJsxEZKIoC9nZ2URHR/P111/TvXt3t8Mx\nxhjXWKLI5YMPPmDz5s088MAD9OzZk40bN1p9JmNMhReZYxSl3PW0f/9+hgwZwm9/+1veeecdK+Jn\njDG5RGiiKJ1Moaq89tprtGzZkg8//JD/+7//Y+nSpVbEzxhjconIS+bSalAkJiYyfvx4EhISmDVr\nFhdeeGEpbdkYY8qPCG1RlPy7fr+fTz75BHCK+C1ZsoRFixZZkjDGmAJEZqIoYZti27ZtXH755fTv\n35+vv/4agISEBCviZ4wxhYjMRFHMPJGdnc0TTzxB27ZtWb9+PS+//LLd8mqMMUGqEGMUAwYMYMGC\nBVx33XU899xz1KtXLyRxGWNMeSSq6nYMxRJ7TjPds3U9davFFbpeeno60dHReL1e3n33XQAGDx5c\nFiEaY0zYEZFVqppQku9GZNdTUU2KJUuW0L59+5wifoMHD7YkYYwxJRSRiaKgwewTJ05wxx130K1b\nN9LT02nRokUZR2aMMeVPZI5R5JMnvv76a0aPHk1iYiKTJ0/mscceo0qVKmUfnDHGlDMRmSgKEh8f\nzzfffEPXrl3dDsUYY8qNiBzM3rd9IzUrx/Dee++xZcsW/vCHPwDg8/nsmQhjjMlH2A5mi0hfEdkq\nIttF5P58Po8VkTcDny8TkSbBbHf//n1cf/31DB48mPfffz+niJ8lCWOMKX0hSxQi4gWeA/oBLYHh\nItIyz2o3A8mqegHwd+CJorbrSz1G54vb8dFHH/H444/z7bffWhE/Y4wJoVC2KDoC21V1p6pmAnOA\nQXnWGQS8Gnj9DtBLiigN6zt2kBYtWrF27Vruv/9+oqOjSz1wY4wxp4VyMLs+sCfXchLQqaB1VDVb\nRFKA2sCh3CuJyERgYmAxY+l3SzZYET8A6pDnWFVgdixOs2Nxmh2L035T0i+GMlHk1zLIO3IezDqo\n6gxgBoCIrCzpgEx5Y8fiNDsWp9mxOM2OxWkisrKk3w1l11MS0DDXcgPgp4LWEZEooDpwJIQxGWOM\nKaZQJooVQDMRaSoiMcAwYG6edeYCowOvrwe+1Ei7X9cYY8q5kHU9BcYcJgOfAV5gtqpuFJFHgJWq\nOheYBbwmIttxWhLDgtj0jFDFHIHsWJxmx+I0Oxan2bE4rcTHIuIeuDPGGFO2IrIooDHGmLJjicIY\nY0yhwjZRhKr8RyQK4ljcLSKbRGSdiHwhIo3diLMsFHUscq13vYioiJTbWyODORYiMiTwu7FRRN4o\n6xjLShB/I41EZKGIrA78nfR3I85QE5HZInJARDYU8LmIyLOB47RORDoEtWFVDbsfnMHvHcB5QAyw\nFmiZZ53bgBcDr4cBb7odt4vHoicQH3g9qSIfi8B6VYFFwFIgwe24Xfy9aAasBmoGluu6HbeLx2IG\nMCnwuiXwo9txh+hYdAc6ABsK+Lw/8AnOM2ydgWXBbDdcWxQhKf8RoYo8Fqq6UFVTA4tLcZ5ZKY+C\n+b0A+AvwJJBelsGVsWCOxQTgOVVNBlDVA2UcY1kJ5lgoUC3wujq/fqarXFDVRRT+LNog4N/qWArU\nEJFzitpuuCaK/Mp/1C9oHVXNBk6V/yhvgjkWud2Mc8VQHhV5LETkIqChqn5UloG5IJjfi+ZAcxFZ\nIiJLRaRvmUVXtoI5Fn8CbhKRJGA+8LuyCS3sFPd8AoTvxEWlVv6jHAj63ykiNwEJQI+QRuSeQo+F\niHhwqhCPKauAXBTM70UUTvfT5TitzG9EpLWqHg1xbGUtmGMxHHhFVZ8WkS44z2+1VlV/6MMLKyU6\nb4Zri8LKf5wWzLFARHoDDwLXqGpGGcVW1oo6FlWB1sBXIvIjTh/s3HI6oB3s38iHqpqlqruArTiJ\no7wJ5ljcDLwFoKrfAXE4BQMrmqDOJ3mFa6Kw8h+nFXksAt0t03GSRHnth4YijoWqpqhqHVVtoqpN\ncMZrrlHVEhdDC2PB/I18gHOjAyJSB6crameZRlk2gjkWiUAvABFpgZMoDpZplOFhLjAqcPdTZyBF\nVX8u6kth2fWkoSv/EXGCPBZ/A6oAbwfG8xNV9RrXgg6RII9FhRDksfgM6CMimwAfcK+qHnYv6tAI\n8ljcA7wkInfhdLWMKY8XliLyX5yuxjqB8ZiHgWgAVX0RZ3ymP7AdSAXGBrXdcnisjDHGlKJw7Xoy\nxhgTJixRGGOMKZQlCmOMMYWyRGGMMaZQliiMMcYUyhKFCTsi4hORNbl+mhSybpOCKmUWc59fBaqP\nrg2UvPhNCbZxq4iMCrweIyLn5vpspoi0LOU4V4hI+yC+83sRiT/TfZuKyxKFCUdpqto+18+PZbTf\nG1W1HU6xyb8V98uq+qKq/juwOAY4N9dn41V1U6lEeTrO5wkuzt8DlihMiVmiMBEh0HL4RkS+D/xc\nms86rURkeaAVsk5EmgXevynX+9NFxFvE7hYBFwS+2yswh8H6QK3/2MD7f5XTc4A8FXjvTyIyRUSu\nx6m59Xpgn5UCLYEEEZkkIk/minmMiPyzhHF+R66CbiLygoisFGfuiT8H3rsDJ2EtFJGFgff6iMh3\ngeP4tohUKWI/poKzRGHCUaVc3U7vB947AFypqh2AocCz+XzvVuAfqtoe50SdFCjXMBToGnjfB9xY\nxP6vBtaLSBzwCjBUVdvgVDKYJCK1gN8CrVS1LfBo7i+r6jvASpwr//aqmpbr43eA63ItDwXeLGGc\nfXHKdJzyoKomAG2BHiLSVlWfxanl01NVewZKeTwE9A4cy5XA3UXsx1RwYVnCw1R4aYGTZW7RwL8C\nffI+nLpFeX0HPCgiDYD3VPUHEekFXAysCJQ3qYSTdPLzuoikAT/ilKH+DbBLVbcFPn8VuB34F85c\nFzNF5GMg6JLmqnpQRHYG6uz8ENjHksB2ixNnZZxyFblnKBsiIhNx/q7PwZmgZ12e73YOvL8ksJ8Y\nnONmTIEsUZhIcRewH2iH0xL+1aREqvqGiCwDBgCfich4nLLKr6rqA0Hs48bcBQRFJN/5TQK1hTri\nFJkbBkwGrijGv+VNYAiwBXhfVVWcs3bQceLM4vZX4DngOhFpCkwBLlHVZBF5BafwXV4C/E9Vhxcj\nXlPBWdeTiRTVgZ8D8weMxLma/gUROQ/YGehumYvTBfMFcL2I1A2sU0uCn1N8C9BERC4ILI8Evg70\n6VdX1fk4A8X53Xl0HKfseX7eA67FmSPhzcB7xYpTVbNwupA6B7qtqgEngRQRORvoV0AsS4Gup/5N\nIhIvIvm1zozJYYnCRIrngdEishSn2+lkPusMBTaIyBrgQpwpHzfhnFAXiMg64H843TJFUtV0nOqa\nb4vIesAPvIhz0v0osL2vcVo7eb0CvHhqMDvPdpOBTUBjVV0eeK/YcQbGPp4GpqjqWpz5sTcCs3G6\ns06ZAXwiIgtV9SDOHVn/DexnKc6xMqZAVj3WGGNMoaxFYYwxplCWKIwxxhTKEoUxxphCWaIwxhhT\nKEsUxhhjCmWJwhhjTKEsURhjjCnU/wOcEcvMmm1JAQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f81cc686f98>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_roc_curve(fpr, tpr, label=None):\n",
    "    plt.plot(fpr, tpr, linewidth=2, label=label)\n",
    "    plt.plot([0, 1], [0, 1], 'k--')\n",
    "    plt.axis([0, 1, 0, 1])\n",
    "    plt.xlabel('False Positive Rate')\n",
    "    plt.ylabel('True Positive Rate')\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.95897170783471819"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ROC AUCの計算\n",
    "from sklearn.metrics import roc_auc_score\n",
    "roc_auc_score(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "forest_clf = RandomForestClassifier(random_state=42)\n",
    "y_probas_forest = cross_val_predict(forest_clf, X_train, y_train_5, cv=3,\n",
    "                                    method='predict_proba')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y_scores_forest = y_probas_forest[:, 1] # score = 陽性クラスの確率\n",
    "fpr_forest, tpr_forest, thresholds_forest = roc_curve(y_train_5,y_scores_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEKCAYAAAAMzhLIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8VGXWwPHfSUgIHZVFXUBAROmEJoLSREWK4isgRTCA\nggpYwbU31rWtuooiCFKUVbGC4KIgiPQOoQvSCYoCAtISUs77xzMpxJBMQiY3k5zvZ7POM3Pn3pNL\n5p65z3PveURVMcYYY84mxOsAjDHG5G+WKIwxxmTKEoUxxphMWaIwxhiTKUsUxhhjMmWJwhhjTKYC\nlihEZLyI/C4iG87yuojICBHZJiLrRKRhoGIxxhiTc4E8o5gI3JjJ6+2B6r6fgcCoAMZijDEmhwKW\nKFR1PvBHJot0Bj5UZylQVkQuDlQ8xhhjcqaIh9uuAOxN047xPfdr+gVFZCDurIMSJUo0qlGjRp4E\nmBdUISEpiSR1j33Ppvn/1Adn3kOfZhlN/2xqI/170q/zr+tNG0e692S4zvRxasavZ7q9v1YHyPx3\n920lk3VmHOeZK8v8907zDr/31Zm/fUZFD/76e2Xwu2f5e2X0l4Df+yrD9ZoCK+Ho7yTFnYCkxIOq\n+recrMPLRCEZPJfh36+qjgHGADRu3FhXrlwZyLjOycnTCRw6fpo/Trifg8fjUh4fOuO/cfxx/DQn\nTid6HXLApf+Hzugf3nhPBEJEENx/cf9zz6V5nPJ8iFtWRAgRAPGtAwT3nIj71w4Jcc+l3QZpHic/\nT5r1nbms7zkAFeLioGhRoUgR0CQ4ckQoWwaKhgsnTsCRI3DRhULRovDHIeGPQ1C1KkRECEePwL59\nUOMK9/qhQ0LMHqhbVwgPh19/gd27hWbNoEiosHs37NwBba8VQkJg1SrhwO/QsaOLb9Ei+GUf9Ojh\nfr9vpgtHj0LUHe6XnDAewsKEfn3d7/TBRLed3r3d7/buSKF8eejR3e3Tl1+GKy6Hrl3d66++Cg0b\nCO3aQVKiMHkyNGoEdesIsbHwwxwhsj5ccolrHzgAF5WHiGKQEB/C9I8+JDH+EF+MfWN3jv82Alnr\nSUSqAN+oap0MXnsP+FFVP/G1twCtVfUvZxRp5WWiUFWOxyWkHtx9CeCg7yCf9uDvHscRG5+UrW2E\nhQrnFQ+nZESRMz6kvs9M6of0jA/VmR9ckdQPYfLjtB9MSfdhPOs6zlg29YMp6bYRkmYb6Q8OZ37g\nMzgQnHEQyehAkGZ9yQeZDGJN+3za19PGmryN9Psho32T8Tqy2jfp9rmkvvfMA2dGB8Mz/+1C/NmG\n37H68feR7u8ieV+f+fdPyr4+fhxOnoTy5V17zx7480+o4/tkz5zpXv+//3PtL7907731Vtd+6y24\n4ALo3du177sP6tWDAQPgs89g5Ejo3h0GDYKEBLj4YnjsMRg61K23XDn45z9de+tWuOIK+Ogj6NUL\nFiyAli3hiy+gSxeYMcMdxBctgubN3fPdusGmTVCzJnz8Mdx5J2zcCJdeCh9+CA88ABs2QIUKMGYM\nvPIKrF8PxYvDe+/BO+/A6tUQFgajR7vfZ/Nm97t8+KFb98svu/Znn8Eff8A997j2559DXFzq7z5n\njltPy5auvWoVlCzpfieAw4ehWDGIiMjwkJGlffv2ce+999K9e3duv/32lOdFZJWqNs7JOr1MFB2B\nIUAHoCkwQlWvzGqd55IoVJU/TyVw6ETcXw7yZ3zzT3NGcDoxewf+8CIhXFAinPNLhHNByaIpj88v\nEZ7m+XDOL1GU80uEUzqiSIYfUmPSU1/3ZEiIO/DMmQN160KlSu4b9DffQIsWULky7N0LkyZBz57u\nm/SGDe6A98gjcMkl7uD04IPuIHrhhe5A/5//uPf87W/uoPf44xAbC0WLugPpBx+47QBERcH8+bBz\np2u3aQPbt8OuXS6+li0hNBTmznWvN2wI1avDp5+69jXXuFhfesltY+xYCA+Hu++GpCSXSDp1gvbt\nIT4ennwSOnSA1q3h2DH43/+gWTP3ux4/DrNnQ6tWcN55LrH88ov7PcPDITHR9+WggH/MVJX333+f\nYcOGER8fz3/+8x/uvvvulNfPJVEErOtJRD4BWgPlRCQGeBYIA1DV0cAMXJLYBpwE+gUqlp9/O8aL\nMzaz4OeDJCRlLzEWCwtNc3B3P+VKFv3rwb9EUc4vGU6J8FA78BdQqnD6tDv4iMDRo7B7t/tmDO7g\n+/vv7uAGsHChayd/q37zTShSBIYMgWXL4I033AH00Ufd6506wZVXwjPPuINntWrum+/w4e710FB3\nwPznP9031o4d3Tfff/zDHRj79IHJk1MTxZNPui6KqlXd6x99BP36uQPogQOwY4f7Vn3hhe73OnLE\nHVTBve+RR+DgQfctu1s3l5SS3X9/6jdkgAkTUpMYwPTpqY/BfRtPa+HC1McRES4xJAsJcWcYycLC\nXPdLslKloEeP1HbJknDLLant4sXhsstS26GhFHjbt29nwIABzJ07lzZt2jB27FiqVauWa+sP6BlF\nIGTnjOLoqXjenL2VD5fsJtGXIEoWLfKXg/z5Jd3j5IN92oN/sfBC8FdWSKi6b+IhIe5gf+qU6yJp\n2dIdPLdtcweowYPdgWbmTHdAnDMHKlaEhx5yB/uTJ13XwH33uW6I+Hi3/v794fvv3UEa4PbbXULY\nts21b7gBtmxx37qPHXMH7s2bYcQI93rfvhAZ6b7pg0sSnTqlducMH+6+iV97rTuwL1jgksCll7r2\n7t2uy6ZkSXfAT0hwB9kQu622wJs6dSpRUVG89tpr3HXXXRl+WT2XMwpUNah+GjVqpFlJSEzSj5ft\n1gbDZ2nlR7/RKo99o49/tU4PHIvN8r3Ge4mJqY8PHFBdsiS1/fHHqm+/ndru31/1iSdS2zVrqj75\nZGq7enXVxx5zj5OSXOfNM8+4dkyMa7/8smsvWqRaurTqjz+69oIFqrVrq65d69rLlqkOGqQaF+fa\nK1eqvvNO6rb27FHdtCm1vX+/ey5ZfLyLwZjcsH79ev3ggw9S2gcPHsx0eWCl5vC46/mBP7s/WSWK\n5TsPaYe35mvlR7/Ryo9+o91GLdYN+45k+h5z7pIP7qdPq0ZHqx465Nq//6765Zeqv/zi2jt2qPbt\n6w7SqqozZqhedJHqH3+49rPPur/K5APqPfeoXnyxO8iqqt54o2qFCqqnTrl2t26qt92mGuv7DjBw\noOo//pEa17Bhqp9+mtp++WXV+fNTY124UPW333JtNxgTcHFxcfrss89qWFiYVqxYUU8lfxiyYIlC\nVX85clLv+3h1SoJo9uJsnRa9T5PsK5xf0u6m7dtTv8UnJrpv7KNHu/ahQ6qNGqm++qprx8Wlfks/\nfVp11SrXHjfOvb5woWt/+aVrf/+9a0+f7tqzZqk2b576rX3uXNXnn09NPCtWuAN9cvvPP1OThDGF\nzdKlS7V27doKaO/evfXAgQN+v/dcEkXQj1EkJimjftzGyLnbORWfSNEiIdzdqhr3tqpm4ws+yf3Z\njRpB2bKuH/3zz13feESEGxicPdsNXAK0besGOrdudYO2zZu7gdVPPnGvX3ed6zcfPNj1+z/5JNSv\nD7fd5vrrx4xxfetVqrhxgGXL3KBt6dLuipakJDeoa4zx3759+6hatSoXXngho0ePpmPHjtl6f769\nPDYQ0ieK0fO28/K3PwHQoe5FPN6+JpXOL+5VeJ6ZPt0dmOvWdddhV6rkLoe8/XaYNQvatYMffnCX\nMX72mbtmff9+d8XL1KlumXffdetaudJdrdOhg2u7m5s8+9WMKdS2bt3K5ZdfDsCUKVNo27YtpUuX\nzvZ6ziVRBPX1ECdPJzBm/g4A3uoRybu3NypQSSIpyX1jj49315u/+GLqa926uatmDh1Kbf/zn+5x\n6dLu6pgk3y0gTZq4q3lq13btW25xV8VceGFqOzlJADRunJokwJKEMV44cuQIAwcOpEaNGsyfPx+A\n//u//8tRkjhXQZ0oPlm+lz9OnKZ+xTLcXP/vXodzTo4ccWcAP//s2h9+6K7/3r3bddMkX0OfrFo1\nd3BPThTz5sG//uUeh4bCd9+56+rB3YQ0aFDqXbXh4XbJpDH52bRp06hduzbjxo3jkUceoUmTJp7G\nE7SHi7iERMbM3w7AkGurB81Nbsk9fVu3uhu1kg/ux465W/4//NC169eHZ59118SLwLBhZxaLe/ll\nV6LAd0ZK06buzldjTHC766676Ny5MxdccAHLli3jlVdeoVixYp7GFLRDil+siuG3P+OocVEp2tYo\n73U4Z7V7N5x/PpQo4W7QGj/eJYVSpdwNVd9955arVMmdFTRo4Nr167ufZHYGYEzBlTxWLCI0btyY\nypUr8+ijjxIeHu5xZE5QHn7iE5MY9WPy2cRlhITkj7MJVTcwvHixa69Y4QaYV692B/qbb3bjBfHx\n7g7aBx6Ab79NfX/Lli6BGGMKj71799KpUyf++9//AnDPPffw9NNP55skAUGaKKZF/0LM4VNc+rcS\ntK/j7VxHmzalDgSLuIP/f/7j2tWrw2uvuRIL4K48mj/flX8wxhRuSUlJjBo1itq1a/Pjjz8SFxfn\ndUhnFZSJYu6W3wHod3VVQvP4bELVXT66w11sxX//6+4n2L/ftT/9NDVxlC3ryiJXqpSnIRpj8rmf\nf/6ZNm3aMGjQIJo2bcqGDRu46667vA7rrIIyUew6dAKAWhfn3WViyZeaxsa67qOpU137wQdhyZLU\nK4quusqVaTbGmLPZtGkT69atY/z48cyaNYuqVat6HVKmgnIwe9fBkwBULVciT7b32mtuMpTJk11C\n+OQTVxcfXLt8/h1LN8bkE2vXriU6OpqoqCg6d+7Mjh07OO+887wOyy9Bd0aRkORmnSsdUYTziocF\nZBtxca68dEyMazdp4u5UTu5e6tEDLrooIJs2xhQwcXFxPP300zRu3Jinn36a2NhYgKBJEhCEiSIu\n3s2sUrVciYDdO5GU5GofVarkJqdp1crNEJY8QY0xxvhjyZIlNGjQgBdeeIFevXqxZs0aInI6x6mH\ngi5RnE5wgwW53e106pQbX5gxw12VtGiRSxhlyuTqZowxhcS+ffto1aoVx48fZ8aMGXzwwQdccMEF\nXoeVI0GXKOJ8c1hXyaVEcfq0mwmsWDFX+iJ5ysbzziv4c+waY3Lf5s2bAahQoQKfffYZGzdupH3y\n/LhBKugSRW6eUZw44QrezZjh2vPmwVNPnfNqjTGF0OHDh+nfvz+1atViwYIFANxyyy2UKgB30Qbd\nVU9xCUkUB6pccO6JokQJ6NIldc5jmyPBGJMTU6ZMYdCgQRw4cIDHH3/c8yJ+uS3oDo2JSa4mSrlS\nOa99PWsW7NwJAwfCF1/kVmTGmMKof//+TJgwgcjISP73v//RsGFDr0PKdUGbKEoWzXno06e7bqae\nPd3cDcYYkx1pi/hdddVVVK9enWHDhhEWFphL9r0WdGMUSXruieKJJ1zpDUsSxpjs2r17N+3bt2fS\npEkADBw4kMcff7zAJgkIwkQBUDw8NNs1nk6fdvM4//GHq9xq90QYY7IjKSmJkSNHUqdOHRYuXEh8\n8uBmIRCUiaJURPbPJr7+Gk6ehClTAhCQMaZA27JlC61atWLIkCE0b96cDRs2cOedd3odVp4JujEK\nyF63kyrs2ePmlL72WgjS+12MMR7asmULGzduZOLEidxxxx1BM6NmbgnKM4qSEf73BX74oZtiNCHB\nkoQxxn9r1qxhwoQJANx8883s2LGDqKioQpckIEgTRelsdD2FhkJYmEsUxhiTldjYWJ544gmaNGnC\nc889l1LEr2zZsh5H5p2gTBTZ6Xrq3RumTYMgrMNljMljixYtIjIykpdeeok77riD6OjooCzil9sK\nbKL4+Wd45BHYvj0PAjLGBL19+/bRpk0b4uLimDlzJuPHjw+qUuCBFJSJooQfieLbb90VTqdO5UFA\nxpigtWnTJsAV8fvyyy9Zv349N9xwg8dR5S9BmSiKhmUd9v33ww8/QJ06eRCQMSbo/PHHH/Tt25fa\ntWszf/58AG666SZKlizpcWT5T1BeHhsWknmiiI93Bf4uuSSPAjLGBJUvv/ySwYMHc+jQIZ588kmu\nvPJKr0PK14LyjCKru7Kff96V5zhyJI8CMsYEjb59+9K1a1cqVKjAihUreOGFF2zAOgtBeUZRJItE\ncf31ULIkFOKr2YwxaaQt4te8eXNq1qzJ0KFDKWJzC/gloGcUInKjiGwRkW0i8lgGr18iInNFZI2I\nrBORDv6sNzQ080TRqhU89petGWMKo507d3LDDTfw4YcfAq6I36OPPmpJIhsClihEJBQYCbQHagE9\nRaRWusWeAj5T1QZAD+Bdf9ad2RnFtGngm1zKGFOIJSYmMmLECOrUqcPSpUtTzipM9gXyjOJKYJuq\n7lDV08BkoHO6ZRRILvZdBvjFnxWHnmUwWxUGD4Z33slZwMaYgmHz5s20aNGCBx54gFatWrFx40b6\n9u3rdVhBK5DnXhWAvWnaMUDTdMs8B8wSkfuAEsB1Ga1IRAYCAwHCL7qMsLN0PYnAxo1w9Oi5BW6M\nCW7btm1jy5YtTJo0idtvv71Q1mfKTYE8o8joXyb9uV9PYKKqVgQ6AJNE5C8xqeoYVW2sqo0h86ue\nSpeGSpVyHrQxJjitWrWK8ePHA+5+iJ07d9K7d29LErkgkIkiBkh7yK7IX7uW7gQ+A1DVJUAEUC6r\nFZ9tjKJfP3jzzZyEaowJVqdOneKxxx6jadOm/POf/0wp4lfaprDMNYFMFCuA6iJSVUTCcYPV09It\nswdoCyAiNXGJ4kBWK85ojEIVYmJg7dpzDdsYEyzmz59P/fr1eeWVV+jbty9r1qyxeyICIGBjFKqa\nICJDgJlAKDBeVTeKyHBgpapOA4YCY0XkIVy3VF/149KEjM4oROC779x/jTEF3759+2jbti2VKlVi\n9uzZtG3b1uuQCqyAXkisqjOAGemeeybN403A1dld79nGKEJDs7smY0ywWb9+PXXr1qVChQpMmTKF\nNm3aUKJECa/DKtCCsoRHRlc9NWsGdvWbMQXXwYMH6dOnD/Xq1Usp4tepUydLEnkgKG9NTD9GkZQE\n7drB5Zd7FJAxJmBUlc8//5whQ4Zw+PBhnn32WZo2TX+lvQmkIE0UZ7ZDQuC55zwJxRgTYFFRUUya\nNInGjRszZ84c6tat63VIhU5QJoqQdCPWK1dCvXoQHu5RQMaYXJW2iF+rVq2oV68eDz74oNVn8khQ\njlGkTxQdOkDz5h4FY4zJVTt27OC6665j4sSJANx5550MGzbMkoSHgjJRpL8EdswYePRRb2IxxuSO\nxMRE3nzzTerWrcuKFSsIyWKCMpN3gjJFS7rqILfc4lEgxphcsWnTJvr378+yZcvo2LEjo0ePpmLF\nil6HZXyCMlGkvY1i3z44eNDNjW33URgTnHbu3Mn27dv5+OOP6dGjh9VnymeC89wuzd/Q++9Dw4Zw\n7Jh34Rhjsm/FihWMHTsWgI4dO7Jjxw569uxpSSIfCspEkXYw+447YPp0m/bUmGBx8uRJhg0bxlVX\nXcVLL72UUsSvVKlSHkdmziYoE0Xa7xtVq7qrnowx+d+PP/5IvXr1eP311xkwYIAV8QsSwZko0pxR\nvPMO+L6QGGPysZiYGK6//noAfvjhB0aPHk2ZMmU8jsr4IygTRfJg9tGjcN998NZb3sZjjDm7tb7a\n/xUrVuTrr79m3bp1tGnTxuOoTHYEZaJIPqEoUQLWr4c+fbyNxxjzVwcOHKBXr15ERkYyb948ADp0\n6EDx4sU9jsxkV1BeHpvc9VSkiLss1hiTf6gqkydP5v777+fo0aM8//zzNGvWzOuwzDnw64xCRMJF\n5LJAB+Ov5BGKJUvgs888DcUYk06fPn3o1asX1apVY82aNTzzzDOEWyG2oJZlohCRjsB64HtfO1JE\npgQ6sMwkXx47diw8+KCXkRhjAJKSklIK+bVp04Y33niDRYsWUbt2bY8jM7nBnzOK4UBT4AiAqkYD\nnp5dJI9RjBgBCxZ4GYkxZtu2bbRt25YJEyYArojfQw89RKiVSigw/EkU8ap6JN1zWc5rHUjJtZ5K\nloRq1byMxJjCKyEhgddee426deuyZs0a614qwPxJFJtF5DYgRESqisibwNIAx5UpEXdp7NNPw+bN\nXkZiTOG0YcMGmjVrxiOPPEK7du3YtGkTvXv39josEyD+JIohQCMgCfgKiAUeCGRQWRGBX3+FF14A\n3yXaxpg8tGfPHnbv3s3kyZOZMmUKf//7370OyQSQJA9AnXUBkVtV9ausnssrRS+urtGrV1Hz4tLE\nx7vnwsK8iMSYwmXZsmWsXbuWgQMHAnD8+HFKlizpcVTGXyKySlUb5+S9/pxRPJXBc0/mZGO5JXkw\nOyzMkoQxgXbixAkefvhhmjVrxquvvkpcXByAJYlC5Kw33IlIO+BGoIKIvJHmpdK4bijPCMLMmfD9\n9/DSS5YsjAmUH374gQEDBrBjxw7uvfdeXn75ZYoWLep1WCaPZXZG8TuwATcmsTHNzyygfeBDO7sQ\ngdWr4b333N3ZxpjcFxMTQ7t27QgNDWXevHm8++67lC5d2uuwjAf8GaOIUNV8U5+16MXVdePa1VxW\nvhSqf50/2xhzbtasWUODBg0A+O6772jVqhXFihXzOCpzrgI9RlFBRCaLyDoR2Zr8k5ON5R6XHSxJ\nGJN7fvvtN7p3707Dhg1TivjdeOONliSMX4liIjABd3RuD3wGTA5gTFkSgSefhNGjvYzCmIJBVfnv\nf/9LrVq1mDp1Ki+88ALNmzf3OiyTj/iTKIqr6kwAVd2uqk8BnheTX7zYlRg3xpybXr160adPH664\n4gqio6N58sknCbMrREwa/gwFx4mr671dRO4B9gHlAxtW5gSYO9fLCIwJbklJSYgIIsINN9xAs2bN\nGDx4sNVnMhny54ziIaAkcD9wNTAA6B/IoIwxgbN161batGnD+PHjAejXrx/333+/JQlzVlkmClVd\npqrHVHWPqvZR1ZuB3XkQ21mdOCF06QKzZ3sZhTHBJSEhgVdffZX69euzbt06G6Q2fss0UYhIExG5\nRUTK+dq1ReRDPC4KePIkbNkChw97GYUxwWPdunVcddVVPProo7Rv355NmzbRq1cvr8MyQSKzO7Nf\nAroAa4GnfJMVPQC8AtyTN+FlrHx52LDBywiMCS4xMTHs3buXzz//nC5duqRMJ2yMPzIbzO4M1FfV\nUyJyPvCLr73F35WLyI3AW0Ao8L6qvpzBMrcBz+HmuFirqll+zYmNhVOnwM6cjTm7xYsXs27dOu65\n5x46dOjAjh07KFGihNdhmSCUWddTrKqeAlDVP4CfspkkQoGRuHsvagE9RaRWumWqA48DV6tqbcCv\niU1nzYSWLd2cFMaYMx0/fpwHHniAa665htdffz2liJ8lCZNTmZ1RXCoiyaXEBaiSpo2q3prFuq8E\ntqnqDgARmYw7S9mUZpkBwEhVPexb5+/+BF2yFBw6BPZ3b8yZZs2axcCBA9mzZw+DBw/mxRdftCJ+\n5pxllii6pGu/k811VwD2pmnH4ObeTutyABFZhOueek5Vv0u/IhEZCAwECL/oMq5rC3fuyGY0xhRw\ne/fupWPHjlSrVo358+dzzTXXeB2SKSDOmihUdc45rjuj0bL0FQiLANWB1kBFYIGI1Ek/R7eqjgHG\ngCsKKBmu2pjCadWqVTRq1IhKlSoxY8YMWrRoQUREhNdhmQLEnxvucioGqJSmXRE3IJ5+ma9VNV5V\ndwJbcIkjU889D3ffnWtxGhOU9u/fT7du3WjcuHFKEb/rr7/ekoTJdYFMFCuA6iJSVUTCgR7AtHTL\nTMVXN8p3r8blQJadSn/+CdbtagorVeWDDz6gVq1aTJ8+nRdffNGK+JmA8nvaHxEpqqpx/i6vqgki\nMgSYiRt/GK+qG0VkOLBSVaf5XrtBRDYBicAjqnooq3W/8TqEnfY3EmMKlh49evDZZ59x9dVX8/77\n71OjRg2vQzIFnD8TF10JjAPKqOolIlIfuEtV78uLANMrenF13bZxLZXOL+7F5o3xRNoifh988AHH\njh1j0KBBhIQEslPAFCSBnrhoBNAJOASgqmvxuMx4x47w+uteRmBM3vnpp59o2bIl48aNAyAqKooh\nQ4ZYkjB5xp+/tBBVTV8EMDEQwfjrssvgoou8jMCYwIuPj+fFF1+kfv36bNq0iZIlS3odkimk/Bmj\n2OvrflLf3db3AZ5OhTpiBFQ8z8sIjAms6Oho+vXrR3R0NF27duXtt9/mIvt2ZDziT6K4F9f9dAnw\nGzDb95wxJkD279/P/v37+fLLL7n11qyKIBgTWP4MZp/vq/WULxS9uLqGnF7LR2OLY58fU5AsXLiQ\ndevWMWjQIABOnjxJ8eJ20YbJHYEezF4hIjNEJEpESuVkI7nt+hsEu6fIFBTHjh1jyJAhtGjRgjff\nfDOliJ8lCZNf+DPDXTXgBaARsF5EpopIj4BHlolRo6BDBy8jMCZ3zJw5kzp16vDuu+/ywAMPsHr1\naiviZ/Idv66vU9XFqno/0BD4E/gooFFlwSo9mYJg7969dOrUieLFi7Nw4ULefPNNu7LJ5EtZJgoR\nKSkit4vIdGA5cADwtF5Agwawfr2XERiTM6rK8uXLAahUqRLffvsta9assRIcJl/z54xiA3AV8Kqq\nXqaqQ1V1WYDjylSHDnDBBV5GYEz2/frrr3Tp0oWmTZumFPG77rrrrIifyff8uTz2UlVNCngk2fDi\ni3BxGa+jMMY/qsrEiRN5+OGHiY2N5ZVXXuHqq6/2Oixj/HbWRCEir6vqUOBLEfnLNbR+zHBnjAFu\nu+02vvjiC1q0aMH777/P5Zdf7nVIxmRLZmcUn/r+m92Z7QKuxhVCzM9QKl9crGvMXyUmJiIihISE\ncNNNN3Httddy9913W30mE5TO+lerqst9D2uq6py0P0DNvAkvY716YfdRmHxr8+bNtGjRIqWI3x13\n3MG9995rScIELX/+cvtn8NyduR1Idjz3HISFeRmBMX8VHx/PCy+8QGRkJFu2bKFMGRtIMwVDZmMU\n3XGz0lWno6+MAAAgAElEQVQVka/SvFQKOJLxu/JGFlVHjMlza9asoW/fvqxbt47u3bszYsQIypcv\n73VYxuSKzMYoluPmoKgIjEzz/DFgTSCDykqDBvBrlhOmGpN3fvvtNw4ePMjUqVPp3Lmz1+EYk6vO\nmihUdSewE1ctNl+55x6vIzAG5s+fz/r16xk8eDA33ngj27Zto1ixYl6HZUyuO+sYhYjM8/33sIj8\nkebnsIh4Wk3WEoXx0p9//smgQYNo1aoVI0aMSCniZ0nCFFSZDWYnT3daDvhbmp/ktmfi473cuinM\nZsyYQe3atXnvvfd4+OGHrYifKRQyuzw2+W7sSkCoqiYCzYC7gRJ5ENtZ3XSTl1s3hdXevXvp3Lkz\nZcqUYfHixbz++uuUKOHpR8GYPOHP5bFTcdOgVgM+xN1D8XFAo8pCv35ebt0UJqrK0qVLAVfEb9as\nWaxevZqmTZt6HJkxecefRJGkqvHArcCbqnofUCGwYWWuR3cvt24Ki19++YVbbrmFZs2apRTxa9Om\nDeHh4R5HZkze8idRJIhIN6AP8I3vOU9vd4uN83LrpqBTVd5//31q1arFrFmzeO2116yInynU/L0z\nuw2uzPgOEakKfBLYsDLXs6eXWzcFXdeuXRkwYACRkZGsX7+eoUOHUqSIP4WWjSmYRP24zVlEigCX\n+ZrbVDUhoFFloujF1XXSe+u57WYr9mRyT9oifpMmTeLkyZMMGDDA6jOZAkNEVqlq45y8158Z7loA\n24BxwHhgq4h4eh7eurWXWzcFzYYNG7j66qtTivj16dPHKr0ak4Y/n4T/AB1U9WpVbQ50BN4KbFiZ\nO3HCy62bguL06dM8//zzNGzYkO3bt3Peeed5HZIx+ZI/iSJcVTclN1R1M+DpZR+vve7l1k1BsGrV\nKho1asRzzz1Ht27d2LRpE127dvU6LGPyJX9G6FaLyHvAJF/7djwuCtgkR71sxqQ6dOgQR44cYfr0\n6XTq1MnrcIzJ17IczBaRCOB+4BpAgPnA26oaG/jw/qroxdV179b1lC9lg9kme+bOncv69eu5//77\nAYiNjSXCZsAyhUTABrNFpC5wIzBFVW9W1ZtU9d9eJYlkx/70cusm2Bw9epS7776ba6+9llGjRqUU\n8bMkYYx/Mqse+wSufMftwPciktFMd5744AOvIzDBYvr06dSqVYv333+fYcOGsWrVKiviZ0w2ZTZG\ncTtQT1VPiMjfgBm4y2M9d911XkdggsHevXvp0qULNWrUYOrUqTRp0sTrkIwJSpl1PcWp6gkAVT2Q\nxbJ5qkZNryMw+ZWqsnjxYiC1iN/KlSstSRhzDjI7+F8qIl/5fqYA1dK0v8rkfSlE5EYR2SIi20Tk\nsUyW6yoiKiJ+DbT8edSfpUxhExMTw80338zVV1+dUsSvdevWVsTPmHOUWddTl3Ttd7KzYhEJxc21\nfT0QA6wQkWlp78nwLVcKd1XVMn/XPWEC/Ovp7ERjCrKkpCTGjh3LI488QkJCAm+88QbXXHON12EZ\nU2BkNmf2nHNc95W4ulA7AERkMtAZ2JRuuX8CrwLD/F3xLbecY2SmQOnSpQtTp07l2muvZezYsVx6\n6aVeh2RMgRLIcYcKwN407RjSzWMhIg2ASqr6DZkQkYEislJEVgJUqSK5HasJMgkJCSQluUkYu3Tp\nwtixY5k9e7YlCWMCIJCJIqOjecrdfSISgqsjNTSrFanqGFVtnHyzyJ92H0Whtm7dOpo1a8bYsWMB\n6N27N3fddRci9gXCmEDwO1GISHYvPo/BzbedrCLwS5p2KaAO8KOI7AKuAqb5M6A951w7xUxQiouL\n49lnn6VRo0bs3r2bv/3tb16HZEyh4E+Z8StFZD3ws69dX0Te9mPdK4DqIlJVRMKBHsC05BdV9aiq\nllPVKqpaBVgK3KyqK7NasU1XXPisWLGChg0bMnz4cHr27MnmzZu59dZbvQ7LmELBn6KAI4BOuLu0\nUdW1ItImqzepaoKIDAFmAqHAeFXdKCLDgZWqOi3zNZzd3/+e03eaYHX48GGOHz/OjBkzaN++vdfh\nGFOo+FMUcLmqXikia1S1ge+5tapaP08iTKfoxdV108oNVKtgZRgKuh9++IH169fzwAMPAK7rycpv\nGJMzAZ3hDtgrIlcCKiKhIvIgsDUnG8stNkZRsB05coQBAwbQtm1b3nvvvZQifpYkjPGGP4niXuBh\n4BLgN9yg872BDCorrVp5uXUTSF9//TW1atVi/Pjx/OMf/7AifsbkA1mOUajq77iB6Hzjggu8jsAE\nwp49e+jWrRs1a9Zk2rRpNG5sM1QZkx9kmShEZCxp7n9IpqoDAxKRH/78E8qV9GrrJjepKgsXLqRF\nixZccsklzJ49m6uuusrqMxmTj/jT9TQbmOP7WQSUB+ICGVRWNm30cusmt+zZs4eOHTvSsmXLlCJ+\nLVu2tCRhTD7jT9fTp2nbIjIJ+D5gEfmhppUZD2pJSUmMHj2aRx99FFVlxIgRVsTPmHzMn/so0qsK\nVM7tQLKjTFkvt27O1a233srXX3/N9ddfz5gxY6hSpYrXIRljMuHPGMVhUscoQoA/gLPOLZEXjh2z\nMYpgk5CQQEhICCEhIXTv3p3OnTvTt29fq89kTBDIdIxC3Ke4PvA33895qnqpqn6WF8Gdzc6dXm7d\nZNfatWtp2rQpY8aMAaBnz57069fPkoQxQSLTRKHutu0pqpro+8n8Nu48clk1ryMw/oiNjeWpp56i\ncePGxMTEcNFFF3kdkjEmB/wZo1guIg1VdXXAo/FTiRJeR2Cysnz5cqKiovjpp5+IiorijTfe4Pzz\nz/c6LGNMDpw1UYhIEVVNAK4BBojIduAEbp4JVdWGeRTjX5w8CRfYGEW+9ueff3Lq1Cm+++472rVr\n53U4xphzkNkZxXKgIZDvJh79/XeoVN7rKEx6s2bNYuPGjTz00ENcd911bNmyxcpvGFMAZDZGIQCq\nuj2jnzyKL0MXX+zl1k16hw8fpl+/frRr145x48ZZET9jCpjMzij+JiIPn+1FVX0jAPH4xY4/+cdX\nX33F4MGDOXDgAI8//jjPPPOMJQhjCpjMEkUoUJKM5772VFwcLjLjqT179tCjRw/q1KnDjBkzaNCg\ngdchGWMCILNE8auqDs+zSLLh+HHAKsh6QlWZP38+rVq14pJLLuGHH36gadOmhIWFeR2aMSZAshyj\nyI/Kls23oRVou3fvpn379rRu3TqliN8111xjScKYAi6zRNE2z6LIpiI5qVBlciwpKYl33nmH2rVr\ns3DhQt5++21atGjhdVjGmDxy1kOuqv6Rl4FkR3y81xEULrfccgvTp0+nXbt2vPfee1Su7GlNSGNM\nHgvK7+anT3sdQcEXHx9PaGgoISEh9OzZk65du9KnTx+rz2RMIeTPxEX5TrFiXkdQsK1evZorr7yS\n0aNHA66I3x133GFJwphCKigTRUhQRp3/nTp1iscff5wrr7yS/fv3U6lSJa9DMsbkA0HZ9ZSQ4HUE\nBc/SpUuJiopi69at9O/fn9dee43zzjvP67CMMflAUCaKREsUue7EiRPEx8fz/fffc91113kdjjEm\nHwnKRBFuFSJyxXfffcfGjRsZOnQobdu25aeffiI8PNzrsIwx+UxQ9vbbmOq5OXToEFFRUbRv354P\nPviA077LyCxJGGMyEpSJIinJ6wiCk6ryxRdfUKtWLT7++GOeeuopVqxYYQnCGJOpoOx6skSRM3v2\n7KFXr17Uq1ePWbNmUb9+fa9DMsYEgaA8owgN9TqC4KGq/PDDDwBUrlyZH3/8kaVLl1qSMMb4LSgT\nhY1R+Gfnzp3ccMMNtG3bNqWIX/PmzSlixbKMMdkQlIlC1esI8rfExETeeust6tSpw7Jlyxg1apQV\n8TPG5FhQfrW0M4rMde7cmf/973906NCB0aNH2x3WxphzEpSJwvxV2iJ+ffr0oWfPnvTq1cvqMxlj\nzllAu55E5EYR2SIi20TksQxef1hENonIOhGZIyJWvzoHVq5cSePGjRk1ahQA3bt35/bbb7ckYYzJ\nFQFLFCISCowE2gO1gJ4iUivdYmuAxqpaD/gCeDVQ8RREp06d4tFHH6Vp06YcOHDA5okwxgREIM8o\nrgS2qeoOVT0NTAY6p11AVeeq6klfcylQMYDxFChLliyhfv36vPrqq/Tv359NmzbRqVMnr8MyxhRA\ngRyjqADsTdOOAZpmsvydwLcZvSAiA4GBAOEXXZZb8QW1U6dOkZSUxOzZs2nbNt/OWmuMKQACmSgy\n6iDP8MJWEekNNAZaZfS6qo4BxgAUvbh6ob04dsaMGWzcuJFHHnmEa6+9ls2bNxMWFuZ1WMaYAi6Q\nXU8xQNrrMisCv6RfSESuA54EblbVuADGE7QOHjxI79696dixIx999FFKET9LEsaYvBDIRLECqC4i\nVUUkHOgBTEu7gIg0AN7DJYnfAxhLUFJVJk+eTM2aNfnss8949tlnWb58uRXxM8bkqYB1PalqgogM\nAWYCocB4Vd0oIsOBlao6Dfg3UBL43Hcp5x5VvTlQMQWbPXv2EBUVRf369Rk3bhx169b1OiRjTCEk\nGmT1MIpeXF1/3baR80sUzG/VqsqcOXNSZplbunQpTZo0IdQqIRpjzoGIrFLVxjl5r92ZnY9s376d\nAQMGMHfuXH788UdatWrFVVdd5XVYphCKj48nJiaG2NhYr0Mx2RQREUHFihVzdQzTEkU+kFzE76mn\nniIsLIz33nvPivgZT8XExFCqVCmqVKlid/gHEVXl0KFDxMTEULVq1VxbryWKfOCmm27i22+/pVOn\nTowaNYqKFe2+Q+Ot2NhYSxJBSES44IILOHDgQK6u1xKFR06fPk2RIkUICQmhb9++9OnThx49etgH\n0+Qb9rcYnALx7xaU81EEu+XLl9OoUSPeffddAG677TZ69uxpH0xjTL5kiSIPnTx5kqFDh9KsWTMO\nHz5MtWrVvA7JmHztX//6F7Vr16ZevXpERkaybNkyEhISeOKJJ6hevTqRkZFERkbyr3/9K+U9oaGh\nREZGUrt2berXr88bb7xBUlKSh79F8AvKrqdg/N69cOFCoqKi2LFjB3fffTevvPIKZcqU8TosY/Kt\nJUuW8M0337B69WqKFi3KwYMHOX36NE899RT79+9n/fr1REREcOzYMV5//fWU9xUrVozo6GgAfv/9\nd3r16sXRo0d5/vnnvfpVgp6dUeSR5ImF5s6dy+jRoy1JmKDSujVMnOgex8e79n//69onT7r2p5+6\n9tGjrv3VV6598KBrT5/u2vv3+7fNX3/9lXLlylG0aFEAypUrR9myZRk7dixvv/02ERERAJQqVYrn\nnnsuw3WUL1+eMWPG8M477xBs94zlJ5YoAmj69Om8+qqbYqNNmzZs2rSJ1q1bexuUMUHihhtuYO/e\nvVx++eUMGjSIefPmsW3bNi655BJKlSrl93ouvfRSkpKS+P13qxKUU0HZ9ZTfHThwgAceeIBPPvmE\nyMhIHnzwQcLDwylSxHa3CU4//pj6OCzszHbx4me2y5Q5s12u3Jntiy7yb5slS5Zk1apVLFiwgLlz\n59K9e3eeeOKJM5aZMGECb731FocOHWLx4sVnnR/ezibOjZ1R5CJV5eOPP6ZmzZp88cUXDB8+nGXL\nllkRP2NyKDQ0lNatW/P888/zzjvvMH36dPbs2cOxY8cA6NevH9HR0ZQpU4bExMQM17Fjxw5CQ0Mp\nX758XoZeoFiiyEV79uyhX79+XHbZZaxZs4ann37akoQxObRlyxZ+/vnnlHZ0dDRXXHEFd955J0OG\nDEkpL5KYmJhSej+9AwcOcM899zBkyBC7/PwcWF/IOUpKSuL777+nXbt2VK5cmQULFtCoUSMr4mfM\nOTp+/Dj33XcfR44coUiRIlx22WWMGTOGMmXK8PTTT1OnTh1KlSpFsWLFiIqK4u9//zvgZn+MjIwk\nPj6eIkWK0KdPHx5++GGPf5vgFpTVY/dv28h5+aB67M8//8yAAQOYN28e8+bNo2XLll6HZEyu2Lx5\nMzVr1vQ6DJNDGf37nUv1WOt6yoGEhAT+/e9/U69ePaKjoxk3bpwV8TPGFFjW9ZQDnTp1YubMmXTu\n3Jl333035ZTXGGMKIksUfoqLiyMsLIyQkBDuuusu+vfvT7du3WyAzBhT4FnXkx+WLl1Kw4YNGTly\nJABdu3bltttusyRhjCkULFFk4sSJEzz00EM0b96cY8eOUb16da9DMsaYPGddT2exYMECoqKi2Llz\nJ4MGDeKll16idOnSXodljDF5zs4oziIhIYGwsDDmzZvHyJEjLUkYk8eSy4XXqVOHm266iSNHjuTK\nenft2kWdOnVyZV1pPffcc1SoUCGl9Pljjz2W69tIFh0dzYwZMwK2/vQsUaQxdepUXnrpJcAV8du4\ncaPdG2GMR5LLhW/YsIHzzz8/ZYwwP3vooYeIjo4mOjqal19+2e/3na38yNnkdaKwrifgt99+4777\n7uPzzz+nYcOGDB061Ir4GeNT5bH/BWS9u17u6PeyzZo1Y926dYC7Y7tz584cPnyY+Ph4XnjhBTp3\n7syuXbto374911xzDYsXL6ZChQp8/fXXFCtWjFWrVtG/f3+KFy/ONddck7Le2NhY7r33XlauXEmR\nIkV44403aNOmDRMnTmTq1KkkJiayYcMGhg4dyunTp5k0aRJFixZlxowZnH/++X7FPmfOHIYNG0ZC\nQgJNmjRh1KhRFC1alCpVqtC/f39mzZrFkCFDaNKkCYMHD+bAgQMUL16csWPHUqNGDT7//HOef/55\nQkNDKVOmDLNnz+aZZ57h1KlTLFy4kMcff5zu3btnb+dnU6E+o1BVJk2aRK1atfj666/517/+xdKl\nS60+kzH5SGJiInPmzOHmm28GICIigilTprB69Wrmzp3L0KFDU6rD/vzzzwwePJiNGzdStmxZvvzy\nS8AVDxwxYgRLliw5Y93JZynr16/nk08+ISoqKqWG1IYNG/j4449Zvnw5Tz75JMWLF2fNmjU0a9aM\nDz/8MMNY//Of/6R0Pc2cOZPY2Fj69u3Lp59+yvr160lISGDUqFEpy0dERLBw4UJ69OjBwIEDefvt\nt1m1ahWvvfYagwYNAmD48OHMnDmTtWvXMm3aNMLDwxk+fDjdu3cnOjo64EkCCvkZxZ49e7jrrrto\n3Lgx48aNo0aNGl6HZEy+k51v/rkpuWbTrl27aNSoEddffz3gvuA98cQTzJ8/n5CQEPbt28dvv/0G\nQNWqVYmMjASgUaNG7Nq1i6NHj3LkyBFatWoFQJ8+ffj2228BN/PkfffdB0CNGjWoXLkyW7duBVz3\nc6lSpShVqhRlypThpptuAqBu3bopZzfpPfTQQwwbNiylvXbtWqpWrcrll18OQFRUFCNHjuTBBx8E\nSDnIHz9+nMWLF9OtW7eU98bFxQFw9dVX07dvX2677TZuvfXWc9qnOVXoziiSkpJS/kgqV67MokWL\nmD9/viUJY/KZ5DGK3bt3c/r06ZRv/x999BEHDhxg1apVREdHc+GFF6acBSTPhgduMDwhIQFVPes9\nT5nVuku7rpCQkJR2SEgICQkJfv0OWdXSK1GiBOCOS2XLlk0Z34iOjmbz5s0AjB49mhdeeIG9e/cS\nGRnJoUOH/Np2bipUiWLr1q20bt2aDh06MG/ePAAaN25slV6NycfKlCnDiBEjeO2114iPj+fo0aOU\nL1+esLAw5s6dy+7duzN9f9myZSlTpgwLFy4EXKJJ1rJly5T21q1b2bNnD1dccUWuxV6jRg127drF\ntm3bAJg0aVLKmU1apUuXpmrVqnz++eeASzBr164FYPv27TRt2pThw4dTrlw59u7dS6lSpVLm5MgL\nhSJRJCQk8Morr1CvXj3Wr1/PhAkT7GomY4JIgwYNqF+/PpMnT+b2229n5cqVNG7cmI8++siv3oAJ\nEyYwePBgmjVrRrFixVKeHzRoEImJidStW5fu3bszceLEM84kzlVERAQTJkygW7du1K1bl5CQEO65\n554Ml/3oo48YN24c9evXp3bt2nz99dcAPPLII9StW5c6derQsmVL6tevnzK1cmRkJJ8mT1YeQIWi\nzHi7du2YNWsWt956KyNHjuQif+diNKaQsjLjwS23y4wX2MHs2NhYwsLCCA0NZeDAgQwcOJAuXbp4\nHZYxxgSdAtn1tGjRIiIjI1MGv7p06WJJwhhjcqhAJYrjx49z//3306JFC2JjY+3U2ZhzEGzd0sYJ\nxL9bgUkU8+bNo06dOrzzzjsMGTKEDRs2pFx3bYzJnoiICA4dOmTJIsioKocOHSIiIiJX11ugxiiK\nFy/OggULuPrqq70OxZigVrFiRWJiYjhw4IDXoZhsioiIoGLFirm6zqC+6umrr77ip59+4oknngDc\nrf52T4QxxvzVuVz1FNCuJxG5UUS2iMg2EflLzV0RKSoin/peXyYiVfxZ72+/7adr16506dKFKVOm\ncPr0aQBLEsYYEwABSxQiEgqMBNoDtYCeIlIr3WJ3AodV9TLgP8ArWa038eSfXNWoPt988w0vvfQS\nixcvtiJ+xhgTQIE8o7gS2KaqO1T1NDAZ6Jxumc7AB77HXwBtJYuJqBP/PEDNmrVZu3Ytjz32GGFh\nYbkeuDHGmFSBHMyuAOxN044Bmp5tGVVNEJGjwAXAwbQLichAYKCvGbd0yaINVsQPgHKk21eFmO2L\nVLYvUtm+SJXjIlaBTBQZnRmkHzn3ZxlUdQwwBkBEVuZ0QKagsX2RyvZFKtsXqWxfpBKRlTl9byC7\nnmKASmnaFYFfzraMiBQBygB/BDAmY4wx2RTIRLECqC4iVUUkHOgBTEu3zDQgyve4K/CDBtv1usYY\nU8AFrOvJN+YwBJgJhALjVXWjiAwHVqrqNGAcMElEtuHOJHr4seoxgYo5CNm+SGX7IpXti1S2L1Ll\neF8E3Q13xhhj8laBqfVkjDEmMCxRGGOMyVS+TRSBKv8RjPzYFw+LyCYRWScic0Skshdx5oWs9kWa\n5bqKiIpIgb000p99ISK3+f42NorIx3kdY17x4zNyiYjMFZE1vs9JBy/iDDQRGS8iv4vIhrO8LiIy\nwref1olIQ79WrKr57gc3+L0duBQIB9YCtdItMwgY7XvcA/jU67g93BdtgOK+x/cW5n3hW64UMB9Y\nCjT2Om4P/y6qA2uA83zt8l7H7eG+GAPc63tcC9jlddwB2hctgYbAhrO83gH4FncP21XAMn/Wm1/P\nKAJS/iNIZbkvVHWuqp70NZfi7lkpiPz5uwD4J/AqEJuXweUxf/bFAGCkqh4GUNXf8zjGvOLPvlCg\ntO9xGf56T1eBoKrzyfxetM7Ah+osBcqKyMVZrTe/JoqMyn9UONsyqpoAJJf/KGj82Rdp3Yn7xlAQ\nZbkvRKQBUElVv8nLwDzgz9/F5cDlIrJIRJaKyI15Fl3e8mdfPAf0FpEYYAZwX96Elu9k93gC5N+J\ni3Kt/EcB4PfvKSK9gcZAq4BG5J1M94WIhOCqEPfNq4A85M/fRRFc91Nr3FnmAhGpo6pHAhxbXvNn\nX/QEJqrq6yLSDHf/Vh1VTQp8ePlKjo6b+fWMwsp/pPJnXyAi1wFPAjeralwexZbXstoXpYA6wI8i\nsgvXBzutgA5o+/sZ+VpV41V1J7AFlzgKGn/2xZ3AZwCqugSIwBUMLGz8Op6kl18ThZX/SJXlvvB1\nt7yHSxIFtR8astgXqnpUVcupahVVrYIbr7lZVXNcDC0f8+czMhV3oQMiUg7XFbUjT6PMG/7siz1A\nWwARqYlLFIVxntdpwB2+q5+uAo6q6q9ZvSlfdj1p4Mp/BB0/98W/gZLA577x/D2qerNnQQeIn/ui\nUPBzX8wEbhCRTUAi8IiqHvIu6sDwc18MBcaKyEO4rpa+BfGLpYh8gutqLOcbj3kWCANQ1dG48ZkO\nwDbgJNDPr/UWwH1ljDEmF+XXridjjDH5hCUKY4wxmbJEYYwxJlOWKIwxxmTKEoUxxphMWaIw+Y6I\nJIpIdJqfKpksW+VslTKzuc0ffdVH1/pKXlyRg3XcIyJ3+B73FZG/p3ntfRGplctxrhCRSD/e86CI\nFD/XbZvCyxKFyY9OqWpkmp9debTd21W1Pq7Y5L+z+2ZVHa2qH/qafYG/p3ntLlXdlCtRpsb5Lv7F\n+SBgicLkmCUKExR8Zw4LRGS176d5BsvUFpHlvrOQdSJS3fd87zTPvycioVlsbj5wme+9bX1zGKz3\n1fov6nv+ZUmdA+Q133PPicgwEemKq7n1kW+bxXxnAo1F5F4ReTVNzH1F5O0cxrmENAXdRGSUiKwU\nN/fE877n7sclrLkiMtf33A0issS3Hz8XkZJZbMcUcpYoTH5ULE230xTfc78D16tqQ6A7MCKD990D\nvKWqkbgDdYyvXEN34Grf84nA7Vls/yZgvYhEABOB7qpaF1fJ4F4ROR/4P6C2qtYDXkj7ZlX9AliJ\n++Yfqaqn0rz8BXBrmnZ34NMcxnkjrkxHsidVtTFQD2glIvVUdQSulk8bVW3jK+XxFHCdb1+uBB7O\nYjumkMuXJTxMoXfKd7BMKwx4x9cnn4irW5TeEuBJEakIfKWqP4tIW6ARsMJX3qQYLulk5CMROQXs\nwpWhvgLYqapbfa9/AAwG3sHNdfG+iPwP8LukuaoeEJEdvjo7P/u2sci33uzEWQJXriLtDGW3ichA\n3Of6YtwEPevSvfcq3/OLfNsJx+03Y87KEoUJFg8BvwH1cWfCf5mUSFU/FpFlQEdgpojchSur/IGq\nPu7HNm5PW0BQRDKc38RXW+hKXJG5HsAQ4Nps/C6fArcBPwFTVFXFHbX9jhM3i9vLwEjgVhGpCgwD\nmqjqYRGZiCt8l54A36tqz2zEawo563oywaIM8Ktv/oA+uG/TZxCRS4Edvu6WabgumDlAVxEp71vm\nfPF/TvGfgCoicpmv3QeY5+vTL6OqM3ADxRldeXQMV/Y8I18Bt+DmSPjU91y24lTVeFwX0lW+bqvS\nwJ19QFoAAADUSURBVAngqIhcCLQ/SyxLgauTfycRKS4iGZ2dGZPCEoUJFu8CUSKyFNftdCKDZboD\nG0QkGqiBm/JxE+6AOktE1gHf47plsqSqsbjqmp+LyHogCRiNO+h+41vfPNzZTnoTgdHJg9np1nsY\n2ARUVtXlvueyHadv7ON1YJiqrsXNj70RGI/rzko2BvhWROaq6gHcFVmf+LazFLevjDkrqx5rjDEm\nU3ZGYYwxJlOWKIwxxmTKEoUxxphMWaIwxhiTKUsUxhhjMmWJwhhjTKYsURhjjMnU/wOgp84VI1JD\nqgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f81cbaeb898>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fpr, tpr, \"b:\", label=\"SGD\")\n",
    "plot_roc_curve(fpr_forest, tpr_forest, \"Random Forest\")\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.99289423261842091"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_train_5, y_scores_forest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## 3.4 多クラス分類"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "* 複数の２項分類器を使い最も高いスコアを出力した分類器のクラスを選ぶ　-> OVA法・OVR法\n",
    "* 数字のすべてのペアに対して２項分類器を使う方法 -> OVO法\n",
    "* 大半の場合はOVA法のほうがいい"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 5.])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.fit(X_train, y_train)\n",
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-184958.1444723 , -458365.86837778, -460821.0078114 ,\n",
       "        -138809.18552395, -332941.15023993,  129202.99030501,\n",
       "        -863959.27465755, -252274.22549676, -702436.33851036,\n",
       "        -772441.55572547]])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "some_digit_scores = sgd_clf.decision_function([some_digit])\n",
    "some_digit_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(some_digit_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  1.,  2.,  3.,  4.,  5.,  6.,  7.,  8.,  9.])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.0"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.classes_[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 5.])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.multiclass import OneVsOneClassifier\n",
    "ovo_clf = OneVsOneClassifier(SGDClassifier(random_state=42))\n",
    "ovo_clf.fit(X_train, y_train)\n",
    "ovo_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ovo_clf.estimators_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 5.])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_clf.fit(X_train, y_train)\n",
    "forest_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.1,  0. ,  0. ,  0.1,  0. ,  0.8,  0. ,  0. ,  0. ,  0. ]])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_clf.predict_proba([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.86842631,  0.82699135,  0.86247937])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(sgd_clf, X_train, y_train, cv=3, scoring='accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.90906819,  0.90944547,  0.91023654])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train.astype(np.float64))\n",
    "cross_val_score(sgd_clf, X_train_scaled, y_train, cv=3, scoring='accuracy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## 3.5 誤分類の分析"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5733,    2,   21,   11,   11,   45,   47,    7,   42,    4],\n",
       "       [   1, 6482,   44,   27,    6,   39,    8,   10,  115,   10],\n",
       "       [  58,   34, 5304,  111,   86,   25,   96,   58,  171,   15],\n",
       "       [  50,   43,  125, 5341,    2,  240,   34,   64,  134,   98],\n",
       "       [  17,   24,   33,   10, 5380,    8,   52,   30,   81,  207],\n",
       "       [  81,   47,   37,  184,   81, 4599,  109,   26,  163,   94],\n",
       "       [  31,   26,   46,    3,   43,   99, 5626,    3,   41,    0],\n",
       "       [  25,   19,   67,   25,   55,   11,    3, 5804,   15,  241],\n",
       "       [  51,  145,   71,  141,   14,  167,   59,   23, 5041,  139],\n",
       "       [  44,   34,   27,   86,  166,   29,    3,  220,   75, 5265]])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred = cross_val_predict(sgd_clf, X_train_scaled, y_train, cv=3)\n",
    "conf_mx = confusion_matrix(y_train, y_train_pred)\n",
    "conf_mx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACwlJREFUeJzt3cGLnPUdx/HPJ7tZsokVDe3FXW0MFFtRS3QpaiAH46Gt\nYi49WLFQL7m0GkUQ7cV/QEQPRVhivRj0EHMIWqwF9VCQ0M0msJq1KGpjYsSUUBUhJrv77WEmoCbd\neRbnO89Ovu8XCNn18cfXYd55npl95reOCAGoZU3bAwAYPMIHCiJ8oCDCBwoifKAgwgcKai1827+0\n/S/b79t+tK05mrJ9pe03bM/bfsf2rrZnasL2iO1Dtl9ue5YmbF9me6/td7uP9S1tz9SL7Ye6z4m3\nbb9ge13bM/XSSvi2RyT9WdKvJF0r6be2r21jlhVYkPRwRPxM0s2S/jAEM0vSLknzbQ+xAk9LejUi\nfirp51rls9uekPSApKmIuE7SiKS7252qt7bO+L+Q9H5EfBARZyS9KGlHS7M0EhEnImK2++cv1XlC\nTrQ71fJsT0q6Q9LutmdpwvalkrZJelaSIuJMRPy33akaGZU0bntU0npJn7Q8T09thT8h6eNvfH1M\nqzyib7K9SdIWSQfanaSnpyQ9Immp7UEa2izppKTnui9Pdtve0PZQy4mI45KekHRU0glJn0fEa+1O\n1Vtb4fsC3xuKe4dtXyLpJUkPRsQXbc/z/9i+U9JnEXGw7VlWYFTSjZKeiYgtkr6StKrf/7F9uTpX\nq1dLukLSBtv3tjtVb22Ff0zSld/4elJDcHlke6060e+JiH1tz9PDVkl32f5InZdSt9l+vt2Rejom\n6VhEnLuS2qvOXwSr2e2SPoyIkxFxVtI+Sbe2PFNPbYX/T0k/sX217TF13gzZ39Isjdi2Oq895yPi\nybbn6SUiHouIyYjYpM7j+3pErOozUUR8Kulj29d0v7Vd0pEWR2riqKSbba/vPke2a5W/ISl1Lq0G\nLiIWbP9R0t/UeRf0LxHxThuzrMBWSb+TNGf7cPd7f4qIv7Y408Xofkl7uieEDyTd1/I8y4qIA7b3\nSppV5yc/hyRNtztVb+ZjuUA93LkHFET4QEGEDxRE+EBBhA8U1Hr4tne2PcNKDNu8EjMPwrDN23r4\nkobqAdPwzSsx8yAM1byrIXwAA5ZyA8/GjRtjcnKy0bGnTp3Sxo0bGx07Nzf3fcYCSoiIC30I7ltS\nbtmdnJzUK6+80vd1r7rqqr6vifN1bjkfLll3oGY+Fm3eNculPlAQ4QMFET5QEOEDBRE+UFCj8Idt\nD3wAy+sZ/pDugQ9gGU3O+EO3Bz6A5TUJf6j3wAdwvibhN9oD3/ZO2zO2Z06dOvX9JwOQpkn4jfbA\nj4jpiJiKiKmm994DaEeT8IduD3wAy+v5IZ0h3QMfwDIafTqv+0sj+MURwEWCO/eAgggfKIjwgYII\nHyiI8IGCUjbbtJ2ymVjmHmVr1gzf34HDts/cMP5m5tHRvN8kv7CwkLJuk802h+/ZDuB7I3ygIMIH\nCiJ8oCDCBwoifKAgwgcKInygIMIHCiJ8oCDCBwoifKAgwgcKInygIMIHCiJ8oCDCBwoifKAgwgcK\nInygIMIHCiJ8oKC0vYMztqvO3AL78OHDKevedNNNKetKedtVLy4upqw7MjKSsq6U91gM47brTVyc\n/1cAlkX4QEGEDxRE+EBBhA8URPhAQYQPFNQzfNtX2n7D9rztd2zvGsRgAPI0uYFnQdLDETFr+weS\nDtr+e0QcSZ4NQJKeZ/yIOBERs90/fylpXtJE9mAA8qzoNb7tTZK2SDqQMQyAwWh8r77tSyS9JOnB\niPjiAv9+p6SdfZwNQJJG4dteq070eyJi34WOiYhpSdPd43M+MQGgL5q8q29Jz0qaj4gn80cCkK3J\na/ytkn4n6Tbbh7v//Dp5LgCJel7qR8Q/JHkAswAYEO7cAwoifKAgwgcKInygIMIHCnLG7qS2o/Pj\n/+GRtQPs7OxsyrqSdMMNN6SsOz4+nrLu6dOnU9aVpKznW+bOwEtLS31fc3FxURHR88HgjA8URPhA\nQYQPFET4QEGEDxRE+EBBhA8URPhAQYQPFET4QEGEDxRE+EBBhA8URPhAQYQPFET4QEGEDxRE+EBB\nhA8URPhAQYQPFET4QEFp22v3fdFkWdszZzy+58zNzaWse/3116esu2ZN3nkm63HOnHnt2rV9X/Pr\nr7/W0tIS22sDOB/hAwURPlAQ4QMFET5QEOEDBRE+UFDj8G2P2D5k++XMgQDkW8kZf5ek+axBAAxO\no/BtT0q6Q9Lu3HEADELTM/5Tkh6RtJQ4C4AB6Rm+7TslfRYRB3sct9P2jO2Zvk0HIEWTM/5WSXfZ\n/kjSi5Jus/38dw+KiOmImIqIqT7PCKDPeoYfEY9FxGREbJJ0t6TXI+Le9MkApOHn+EBBoys5OCLe\nlPRmyiQABoYzPlAQ4QMFET5QEOEDBRE+UFDaLrsjIyN9X3dpafjuGB4bG0tbe2FhIWXd/fv3p6y7\nY8eOlHUlaXFxMWXdjJ1wz8mYeXFxURHBLrsAzkf4QEGEDxRE+EBBhA8URPhAQYQPFET4QEGEDxRE\n+EBBhA8URPhAQYQPFET4QEGEDxRE+EBBhA8URPhAQYQPFET4QEGEDxSUtsuu3XOjzxIyHt9zsh7j\nNWtyzgfvvfdeyrqStHnz5pR1M5/HWc8NdtkFcEGEDxRE+EBBhA8URPhAQYQPFET4QEGNwrd9me29\ntt+1PW/7luzBAOQZbXjc05JejYjf2B6TtD5xJgDJeoZv+1JJ2yT9XpIi4oykM7ljAcjU5FJ/s6ST\nkp6zfcj2btsbkucCkKhJ+KOSbpT0TERskfSVpEe/e5DtnbZnbM/0eUYAfdYk/GOSjkXEge7Xe9X5\ni+BbImI6IqYiYqqfAwLov57hR8Snkj62fU33W9slHUmdCkCqpu/q3y9pT/cd/Q8k3Zc3EoBsjcKP\niMOSuIQHLhLcuQcURPhAQYQPFET4QEGEDxRE+EBBadtr933RZFlbSi8tLaWsKw3nzFmOHz+esu7E\nxETKupI0Pj7e9zVPnz6tpaUlttcGcD7CBwoifKAgwgcKInygIMIHCiJ8oCDCBwoifKAgwgcKInyg\nIMIHCiJ8oCDCBwoifKAgwgcKInygIMIHCiJ8oCDCBwoifKCgtF12M3aAHRkZ6fua5ywuLqasmznz\n2bNnU9YdGxtLWXdhYSFlXSlvZ+C33norZV1J2rZtW9/XXFhYYJddABdG+EBBhA8URPhAQYQPFET4\nQEGEDxTUKHzbD9l+x/bbtl+wvS57MAB5eoZve0LSA5KmIuI6SSOS7s4eDECeppf6o5LGbY9KWi/p\nk7yRAGTrGX5EHJf0hKSjkk5I+jwiXsseDECeJpf6l0vaIelqSVdI2mD73gsct9P2jO2Z/o8JoJ+a\nXOrfLunDiDgZEWcl7ZN063cPiojpiJiKiKl+Dwmgv5qEf1TSzbbX27ak7ZLmc8cCkKnJa/wDkvZK\nmpU01/1vppPnApBotMlBEfG4pMeTZwEwINy5BxRE+EBBhA8URPhAQYQPFET4QEFp22t37vVBxjbj\n52Rt3Z21bXfGc+2cdetyPil+5syZlHUl6eDBg31f85577tGRI0fYXhvA+QgfKIjwgYIIHyiI8IGC\nCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYKy\ndtk9KenfDQ//oaT/9H2IPMM2r8TMg7Ba5v1xRPyo10Ep4a+E7ZmImGp1iBUYtnklZh6EYZuXS32g\nIMIHCloN4U+3PcAKDdu8EjMPwlDN2/prfACDtxrO+AAGjPCBgggfKIjwgYIIHyjof6losF9MqPil\nAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f81cc6fdcf8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(conf_mx, cmap=plt.cm.gray)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "row_sums = conf_mx.sum(axis=1, keepdims=True)\n",
    "norm_conf_mx = conf_mx / row_sums"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  9.67921661e-01,   3.37666723e-04,   3.54550059e-03,\n",
       "          1.85716698e-03,   1.85716698e-03,   7.59750127e-03,\n",
       "          7.93516799e-03,   1.18183353e-03,   7.09100118e-03,\n",
       "          6.75333446e-04],\n",
       "       [  1.48323939e-04,   9.61435776e-01,   6.52625334e-03,\n",
       "          4.00474637e-03,   8.89943637e-04,   5.78463364e-03,\n",
       "          1.18659152e-03,   1.48323939e-03,   1.70572530e-02,\n",
       "          1.48323939e-03],\n",
       "       [  9.73481034e-03,   5.70661296e-03,   8.90231621e-01,\n",
       "          1.86304129e-02,   1.44343740e-02,   4.19603894e-03,\n",
       "          1.61127895e-02,   9.73481034e-03,   2.87009063e-02,\n",
       "          2.51762336e-03],\n",
       "       [  8.15527646e-03,   7.01353776e-03,   2.03881912e-02,\n",
       "          8.71146632e-01,   3.26211059e-04,   3.91453270e-02,\n",
       "          5.54558800e-03,   1.04387539e-02,   2.18561409e-02,\n",
       "          1.59843419e-02],\n",
       "       [  2.90996234e-03,   4.10818213e-03,   5.64875043e-03,\n",
       "          1.71174255e-03,   9.20917494e-01,   1.36939404e-03,\n",
       "          8.90106128e-03,   5.13522766e-03,   1.38651147e-02,\n",
       "          3.54330709e-02],\n",
       "       [  1.49418926e-02,   8.66998709e-03,   6.82530898e-03,\n",
       "          3.39420771e-02,   1.49418926e-02,   8.48367460e-01,\n",
       "          2.01069913e-02,   4.79616307e-03,   3.00682531e-02,\n",
       "          1.73399742e-02],\n",
       "       [  5.23825617e-03,   4.39337614e-03,   7.77289625e-03,\n",
       "          5.06928016e-04,   7.26596823e-03,   1.67286245e-02,\n",
       "          9.50659006e-01,   5.06928016e-04,   6.92801622e-03,\n",
       "          0.00000000e+00],\n",
       "       [  3.99042298e-03,   3.03272147e-03,   1.06943336e-02,\n",
       "          3.99042298e-03,   8.77893057e-03,   1.75578611e-03,\n",
       "          4.78850758e-04,   9.26416600e-01,   2.39425379e-03,\n",
       "          3.84676776e-02],\n",
       "       [  8.71645873e-03,   2.47820885e-02,   1.21346778e-02,\n",
       "          2.40984447e-02,   2.39275338e-03,   2.85421296e-02,\n",
       "          1.00837464e-02,   3.93095197e-03,   8.61562126e-01,\n",
       "          2.37566228e-02],\n",
       "       [  7.39620104e-03,   5.71524626e-03,   4.53857791e-03,\n",
       "          1.44562111e-02,   2.79038494e-02,   4.87476887e-03,\n",
       "          5.04286435e-04,   3.69810052e-02,   1.26071609e-02,\n",
       "          8.85022693e-01]])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "norm_conf_mx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADDtJREFUeJzt3X+o1fUdx/HXy3v16tUNh1tBWrNIXCEM7bJygmAOmpu4\nf/aHg4LqDyG35lYw5vqjPyqCGLIRIxDTgmT7Qw3GGPtBW3/sH5lZpnkdjWrdq1fmilVXyuu9970/\n7hFaa57vie/nfO/p/XxAoLevb95cfd7vOed+z/c6IgQglzlNLwCg+wgfSIjwgYQIH0iI8IGECB9I\nqLHwbX/d9t9s/932j5vaoyrbV9v+s+1h26/Y3tH0TlXY7rP9ou3fNL1LFbYX2z5g+1Trc7226Z3a\nsf3D1r+JE7Z/aXt+0zu100j4tvsk/ULSJkk3SvqO7Rub2KUDk5Luj4gbJN0i6bs9sLMk7ZA03PQS\nHfi5pN9FxJckfVmzfHfbSyV9X9JQRKyS1Cdpa7NbtdfUGf8rkv4eEa9FxISkX0n6VkO7VBIRYxFx\ntPXr9zTzD3Jps1tdnu1lkr4paU/Tu1Rh+7OS1kt6UpIiYiIi/t3sVpX0S1pgu1/SoKQzDe/TVlPh\nL5U08qHfj2qWR/RhtpdLWi3pcLObtPUzST+SNN30IhVdJ+mcpH2tpyd7bC9seqnLiYjTkn4q6U1J\nY5LeiYg/NLtVe02F74/5WE9cO2x7kaSDkn4QEe82vc//Y3uzpH9GxAtN79KBfklrJD0REaslnZc0\nq1//sf05zTxavVbSVZIW2r692a3aayr8UUlXf+j3y9QDD49sz9VM9Psj4lDT+7SxTtIW229o5qnU\nrbafaXaltkYljUbEpUdSBzTzhWA2+5qk1yPiXERclHRI0lcb3qmtpsL/q6QVtq+1PU8zL4b8uqFd\nKrFtzTz3HI6IXU3v005E7IyIZRGxXDOf3z9FxKw+E0XEWUkjtle2PrRR0skGV6riTUm32B5s/RvZ\nqFn+gqQ089Cq6yJi0vb3JP1eM6+C7o2IV5rYpQPrJN0h6bjtl1of+0lE/LbBnT6N7pW0v3VCeE3S\nXQ3vc1kRcdj2AUlHNfOdnxcl7W52q/bM23KBfLhyD0iI8IGECB9IiPCBhAgfSKjx8G1va3qHTvTa\nvhI7d0Ov7dt4+JJ66hOm3ttXYudu6Kl9Z0P4ALqsyAU8tnvuqqD+/moXMU5PT2vOnOpfL6enm39j\nXERo5mrSagYGBors0ckOk5OTlf9OJOn999//JCu1NX9+tXtqdLqvVG7niGj7iW7kkt1PqpPgOrVk\nyZIic8+fP19kbknXX399kbl9fX1F5krSiRMnisxdsWJFkbmSdPJk/W9DmJycrHQcD/WBhAgfSIjw\ngYQIH0iI8IGEKoXfa/fAB3B5bcPv0XvgA7iMKmf8nrsHPoDLqxJ+T98DH8D/qnLlXqV74LfendRT\nb1QAsqoSfqV74EfEbrXuLtqL1+oDmVR5qN9z98AHcHltz/g9eg98AJdR6d15rR8awQ+OAD4luHIP\nSIjwgYQIH0iI8IGECB9IqKfuuVfyxpWl7rlX8j5z586dKzJ3fHy8yNyRkZH2B31Cpe7HuGHDhiJz\nJenVV1+tfebU1FSl4zjjAwkRPpAQ4QMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCRE+kBDh\nAwkRPpAQ4QMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QUJHbay9atEg33XRT7XNPnz5d+8xLTp48\nWWTuzp07i8yVyn0+nnvuuSJz77777iJzpXJ/f5s3by4yV5Kefvrp2mdOTExUOo4zPpAQ4QMJET6Q\nEOEDCRE+kBDhAwkRPpBQ2/BtX237z7aHbb9ie0c3FgNQTpULeCYl3R8RR21/RtILtv8YEWWumABQ\nXNszfkSMRcTR1q/fkzQsaWnpxQCU09FzfNvLJa2WdLjEMgC6o/K1+rYXSToo6QcR8e7H/P9tkrZJ\n0sDAQG0LAqhfpTO+7bmaiX5/RBz6uGMiYndEDEXE0Ny5c+vcEUDNqryqb0lPShqOiF3lVwJQWpUz\n/jpJd0i61fZLrf++UXgvAAW1fY4fEX+R5C7sAqBLuHIPSIjwgYQIH0iI8IGECB9IqMhddqenp3X+\n/Pna586fP7/2mZc88sgjReY+8MADReZK0pw5Zb5uT01NFZm7YsWKInMlafny5UXmHj9+vMhcSdq6\ndWvtMw8ePFjpOM74QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKE\nDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8k5IiofWh/f38sXry49rkDAwO1z7xkcHCwyNyx\nsbEicyUVuYW5VO425mvWrCkyV5LOnDlTZO7atWuLzJWkXbvq/6nzt912m44dO9b2h9xyxgcSInwg\nIcIHEiJ8ICHCBxIifCAhwgcSqhy+7T7bL9r+TcmFAJTXyRl/h6ThUosA6J5K4dteJumbkvaUXQdA\nN1Q94/9M0o8kTRfcBUCXtA3f9mZJ/4yIF9oct832EdtHSlz/D6A+Vc746yRtsf2GpF9JutX2Mx89\nKCJ2R8RQRAzZbd8jAKBBbcOPiJ0RsSwilkvaKulPEXF78c0AFMP38YGE+js5OCKel/R8kU0AdA1n\nfCAhwgcSInwgIcIHEiJ8IKGOXtWvavHixdqyZUvtc4eHy71HaHx8vMjcl19+uchcSXr44YeLzN23\nb1+RuZs2bSoyV5L27CnzNpLt27cXmStJjz32WO0zz549W+k4zvhAQoQPJET4QEKEDyRE+EBChA8k\nRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEIu8bPs\nBwYGYtmyZbXPnZiYqH3mJVdccUWRuceOHSsyV5JWrVpVZO769euLzH388ceLzJWkvr6+InNvuOGG\nInMl6cSJE0XmRkTbn1PPGR9IiPCBhAgfSIjwgYQIH0iI8IGECB9IqFL4thfbPmD7lO1h22tLLwag\nnKo/Jvvnkn4XEd+2PU/SYMGdABTWNnzbn5W0XtKdkhQRE5LKXUIHoLgqD/Wvk3RO0j7bL9reY3th\n4b0AFFQl/H5JayQ9ERGrJZ2X9OOPHmR7m+0jto9MT0/XvCaAOlUJf1TSaEQcbv3+gGa+EPyXiNgd\nEUMRMTRnDt8sAGaztoVGxFlJI7ZXtj60UdLJolsBKKrqq/r3StrfekX/NUl3lVsJQGmVwo+IlyQN\nFd4FQJfwZBxIiPCBhAgfSIjwgYQIH0iI8IGEqn4fvyMRoQ8++KD2uRcuXKh95iVLliwpMvfKK68s\nMleSFixYUGTu3r17i8wtdQtsSZqamioyd2RkpMhcSbp48WLtM2+++eZKx3HGBxIifCAhwgcSInwg\nIcIHEiJ8ICHCBxIifCAhwgcSInwgIcIHEiJ8ICHCBxIifCAhwgcSInwgIcIHEiJ8ICHCBxIifCAh\nwgcSKnKX3Xnz5umaa66pfe727dtrn3nJwYMHi8x96KGHisyVpPvuu6/I3GeffbbI3EcffbTIXEk6\ndepUkbljY2NF5krSU089VfvMt956q9JxnPGBhAgfSIjwgYQIH0iI8IGECB9IiPCBhCqFb/uHtl+x\nfcL2L23PL70YgHLahm97qaTvSxqKiFWS+iRtLb0YgHKqPtTvl7TAdr+kQUlnyq0EoLS24UfEaUk/\nlfSmpDFJ70TEH0ovBqCcKg/1PyfpW5KulXSVpIW2b/+Y47bZPmL7yMWLF+vfFEBtqjzU/5qk1yPi\nXERclHRI0lc/elBE7I6IoYgYmjt3bt17AqhRlfDflHSL7UHblrRR0nDZtQCUVOU5/mFJByQdlXS8\n9Wd2F94LQEGV3o8fEQ9KerDwLgC6hCv3gIQIH0iI8IGECB9IiPCBhAgfSMgRUfvQwcHBWLlyZe1z\nx8fHa595yYULF4rM3bBhQ5G5krR1a5k3Sd5zzz1F5o6OjhaZK0lvv/12kbl33nlnkblSuduYR4Tb\nHcMZH0iI8IGECB9IiPCBhAgfSIjwgYQIH0iI8IGECB9IiPCBhAgfSIjwgYQIH0iI8IGECB9IiPCB\nhAgfSIjwgYQIH0iI8IGECB9IqMhddm2fk/SPiod/XtK/al+inF7bV2Lnbpgt+34xIr7Q7qAi4XfC\n9pGIGGp0iQ702r4SO3dDr+3LQ30gIcIHEpoN4e9ueoEO9dq+Ejt3Q0/t2/hzfADdNxvO+AC6jPCB\nhAgfSIjwgYQIH0joP5EZ0Gb8c9TKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f81cb7b90b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "np.fill_diagonal(norm_conf_mx, 0)\n",
    "plt.matshow(norm_conf_mx, cmap=plt.cm.gray)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# cl_a, cl_b = 3, 5\n",
    "# X_aa = X_train[(y_train == cl_a) & (y_train_pred == cl_a)]\n",
    "# X_ab = X_train[(y_train == cl_a) & (y_train_pred == cl_b)]\n",
    "# X_ba = X_train[(y_train == cl_b) & (y_train_pred == cl_a)]\n",
    "# X_bb = X_train[(y_train == cl_b) & (y_train_pred == cl_b)]\n",
    "# plt.figure(figsize=(8,8))\n",
    "# plt.subplot(221); plot_digits(X_aa[:25], images_per_row=5)\n",
    "# plt.subplot(222); plot_digits(X_ab[:25], images_per_row=5)\n",
    "# plt.subplot(223); plot_digits(X_ba[:25], images_per_row=5)\n",
    "# plt.subplot(224); plot_digits(X_bb[:25], images_per_row=5)\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## 3.6 多ラベル分類"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=5, p=2,\n",
       "           weights='uniform')"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "y_train_large = (y_train >= 7)\n",
    "y_train_odd = (y_train % 2 == 1)\n",
    "y_multilabel = np.c_[y_train_large, y_train_odd]\n",
    "knn_clf = KNeighborsClassifier()\n",
    "knn_clf.fit(X_train, y_multilabel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False,  True]], dtype=bool)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y_train_knn_pred = cross_val_predict(knn_clf, X_train, y_multilabel, cv=3)\n",
    "f1_score(y_multilabel, y_train_knn_pred, average=\"macro\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## 3.7 多出力分類"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "noise = np.random.randint(0, 100, (len(X_train), 784))\n",
    "X_train_mod = X_train + noise\n",
    "noise = np.random.randint(0, 100, (len(X_test), 784))\n",
    "X_test_mod = X_test + noise\n",
    "y_train_mod = X_train\n",
    "y_test_mod = X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "knn_clf.fit(X_train_mod, y_train_mod)\n",
    "clean_digit = knn_clf.predict([X_test_mod[some_index]])\n",
    "plot_digit(clean_digit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}