active learning1.ipynb

active_learning1.ipynb

{
  "cells": [
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "# 准备工作"
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "# 若要在binder中执行这个notebook，请先执行下面的指令来安装依赖包\n!pip install scikit-learn numpy pandas matplotlib",
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "假设我们正要完成手写数字识别的任务。我们可以使用著名的mnist数据集来训练这样的机器学习模型。"
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt \nnp.random.seed(1)               # reproduce the result\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom sklearn.datasets import load_digits",
      "execution_count": 1,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "digits = load_digits()\nprint(digits.data.shape)",
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": "(1797, 64)\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "总共有1797个数字，每个数字使用一个64维的向量表示"
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "plt.gray() \nplt.matshow(digits.images[0]) \nplt.show() ",
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<matplotlib.figure.Figure at 0x1c271ddb898>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<matplotlib.figure.Figure at 0x1c26f016630>",
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAC9pJREFUeJzt3V+IXPUZxvHn6Zr4L5HEakUSMV0p\nARFq/hAqAWmTKLFKelNDAgqVluSiFUMLGntTvPNK7EURQtQKxoiJBoq01gQVEVptNsYaTSwaIm6i\nrpJIjIUE49uLOSkxpO7Z7f5+OzPv9wNLZndn5/ntbp45Z2bPnNcRIQC5fGuyFwCgPooPJETxgYQo\nPpAQxQcSovhAQl1RfNvLbb9j+13b6wtnPWJ7xPaekjmn5V1h+0Xbe22/Zfuuwnnn2X7N9htN3n0l\n85rMAduv2362dFaTd8D2m7Z3295ZOGuG7a229zW/w+sKZs1tvqdTb0dtrysSFhGT+iZpQNJ7kgYl\nTZX0hqSrC+ZdL2m+pD2Vvr/LJc1vLk+X9K/C358lTWsuT5H0qqQfFP4efy3pCUnPVvqZHpB0SaWs\nxyT9ork8VdKMSrkDkj6SdGWJ2++GLf4iSe9GxP6IOCHpSUk/KRUWES9LOlzq9s+S92FE7Goufy5p\nr6RZBfMiIo41705p3oodpWV7tqSbJW0slTFZbF+kzobiYUmKiBMR8Vml+KWS3ouI90vceDcUf5ak\nD057f1gFizGZbM+RNE+drXDJnAHbuyWNSNoeESXzHpR0t6SvCmacKSQ9b3vI9pqCOYOSPpH0aPNQ\nZqPtCwvmnW6VpM2lbrwbiu+zfKzvjiO2PU3S05LWRcTRklkRcTIirpU0W9Ii29eUyLF9i6SRiBgq\ncfvfYHFEzJd0k6Rf2r6+UM456jwsfCgi5kn6QlLR56AkyfZUSSskbSmV0Q3FH5Z0xWnvz5Z0aJLW\nUoTtKeqUflNEPFMrt9ktfUnS8kIRiyWtsH1AnYdoS2w/XijrvyLiUPPviKRt6jxcLGFY0vBpe0xb\n1bkjKO0mSbsi4uNSAd1Q/H9I+p7t7zb3dKsk/WmS1zRhbFudx4h7I+KBCnmX2p7RXD5f0jJJ+0pk\nRcS9ETE7Iuao83t7ISJuK5F1iu0LbU8/dVnSjZKK/IUmIj6S9IHtuc2Hlkp6u0TWGVar4G6+1NmV\nmVQR8aXtX0n6qzrPZD4SEW+VyrO9WdIPJV1ie1jS7yLi4VJ56mwVb5f0ZvO4W5J+GxF/LpR3uaTH\nbA+oc8f+VERU+TNbJZdJ2ta5P9U5kp6IiOcK5t0paVOzUdov6Y6CWbJ9gaQbJK0tmtP86QBAIt2w\nqw+gMooPJETxgYQoPpAQxQcS6qriFz78ctKyyCOv2/K6qviSav5wq/4iySOvm/K6rfgAKihyAI/t\nvj4qaObMmWP+muPHj+vcc88dV96sWWN/seLhw4d18cUXjyvv6NGxv4bo2LFjmjZt2rjyDh48OOav\niQg1R++N2cmTJ8f1db0iIkb9wUz6Ibu9aNmyZVXz7r///qp5O3bsqJq3fn3xF7x9zZEjR6rmdSN2\n9YGEKD6QEMUHEqL4QEIUH0iI4gMJUXwgIYoPJNSq+DVHXAEob9TiNydt/IM6p/y9WtJq21eXXhiA\nctps8auOuAJQXpvipxlxBWTR5kU6rUZcNScOqP2aZQDj0Kb4rUZcRcQGSRuk/n9ZLtDr2uzq9/WI\nKyCjUbf4tUdcASiv1Yk4mjlvpWa9AaiMI/eAhCg+kBDFBxKi+EBCFB9IiOIDCVF8ICGKDyTEJJ1x\nqD3ZZnBwsGreeEaE/T8OHz5cNW/lypVV87Zs2VI1rw22+EBCFB9IiOIDCVF8ICGKDyRE8YGEKD6Q\nEMUHEqL4QEIUH0iozQitR2yP2N5TY0EAymuzxf+jpOWF1wGgolGLHxEvS6r7KgoARfEYH0howl6W\ny+w8oHdMWPGZnQf0Dnb1gYTa/Dlvs6S/SZpre9j2z8svC0BJbYZmrq6xEAD1sKsPJETxgYQoPpAQ\nxQcSovhAQhQfSIjiAwlRfCChvpidt2DBgqp5tWfZXXXVVVXz9u/fXzVv+/btVfNq/39hdh6ArkDx\ngYQoPpAQxQcSovhAQhQfSIjiAwlRfCAhig8kRPGBhNqcbPMK2y/a3mv7Ldt31VgYgHLaHKv/paTf\nRMQu29MlDdneHhFvF14bgELazM77MCJ2NZc/l7RX0qzSCwNQzpge49ueI2mepFdLLAZAHa1flmt7\nmqSnJa2LiKNn+Tyz84Ae0ar4tqeoU/pNEfHM2a7D7Dygd7R5Vt+SHpa0NyIeKL8kAKW1eYy/WNLt\nkpbY3t28/bjwugAU1GZ23iuSXGEtACrhyD0gIYoPJETxgYQoPpAQxQcSovhAQhQfSIjiAwn1xey8\nmTNnVs0bGhqqmld7ll1ttX+eYIsPpETxgYQoPpAQxQcSovhAQhQfSIjiAwlRfCAhig8kRPGBhNqc\nZfc826/ZfqOZnXdfjYUBKKfNsfrHJS2JiGPN+fVfsf2XiPh74bUBKKTNWXZD0rHm3SnNGwMzgB7W\n6jG+7QHbuyWNSNoeEczOA3pYq+JHxMmIuFbSbEmLbF9z5nVsr7G90/bOiV4kgIk1pmf1I+IzSS9J\nWn6Wz22IiIURsXCC1gagkDbP6l9qe0Zz+XxJyyTtK70wAOW0eVb/ckmP2R5Q547iqYh4tuyyAJTU\n5ln9f0qaV2EtACrhyD0gIYoPJETxgYQoPpAQxQcSovhAQhQfSIjiAwkxO28cduzYUTWv39X+/R05\ncqRqXjdiiw8kRPGBhCg+kBDFBxKi+EBCFB9IiOIDCVF8ICGKDyRE8YGEWhe/Garxum1OtAn0uLFs\n8e+StLfUQgDU03aE1mxJN0vaWHY5AGpou8V/UNLdkr4quBYAlbSZpHOLpJGIGBrleszOA3pEmy3+\nYkkrbB+Q9KSkJbYfP/NKzM4DeseoxY+IeyNidkTMkbRK0gsRcVvxlQEohr/jAwmN6dRbEfGSOmOy\nAfQwtvhAQhQfSIjiAwlRfCAhig8kRPGBhCg+kBDFBxLqi9l5tWehLViwoGpebbVn2dX+eW7ZsqVq\nXjdiiw8kRPGBhCg+kBDFBxKi+EBCFB9IiOIDCVF8ICGKDyRE8YGEWh2y25xa+3NJJyV9ySm0gd42\nlmP1fxQRnxZbCYBq2NUHEmpb/JD0vO0h22tKLghAeW139RdHxCHb35G03fa+iHj59Cs0dwjcKQA9\noNUWPyIONf+OSNomadFZrsPsPKBHtJmWe6Ht6acuS7pR0p7SCwNQTptd/cskbbN96vpPRMRzRVcF\noKhRix8R+yV9v8JaAFTCn/OAhCg+kBDFBxKi+EBCFB9IiOIDCVF8ICGKDyTkiJj4G7Un/ka/weDg\nYM047dy5s2re2rVrq+bdeuutVfNq//4WLuzvl5NEhEe7Dlt8ICGKDyRE8YGEKD6QEMUHEqL4QEIU\nH0iI4gMJUXwgIYoPJNSq+LZn2N5qe5/tvbavK70wAOW0Hajxe0nPRcRPbU+VdEHBNQEobNTi275I\n0vWSfiZJEXFC0omyywJQUptd/UFJn0h61Pbrtjc2gzW+xvYa2ztt133pGoAxa1P8cyTNl/RQRMyT\n9IWk9WdeiRFaQO9oU/xhScMR8Wrz/lZ17ggA9KhRix8RH0n6wPbc5kNLJb1ddFUAimr7rP6dkjY1\nz+jvl3RHuSUBKK1V8SNityQeuwN9giP3gIQoPpAQxQcSovhAQhQfSIjiAwlRfCAhig8k1Bez82pb\ns2ZN1bx77rmnat7Q0FDVvJUrV1bN63fMzgNwVhQfSIjiAwlRfCAhig8kRPGBhCg+kBDFBxKi+EBC\noxbf9lzbu097O2p7XY3FAShj1HPuRcQ7kq6VJNsDkg5K2lZ4XQAKGuuu/lJJ70XE+yUWA6COsRZ/\nlaTNJRYCoJ7WxW/Oqb9C0pb/8Xlm5wE9ou1ADUm6SdKuiPj4bJ+MiA2SNkj9/7JcoNeNZVd/tdjN\nB/pCq+LbvkDSDZKeKbscADW0HaH1b0nfLrwWAJVw5B6QEMUHEqL4QEIUH0iI4gMJUXwgIYoPJETx\ngYQoPpBQqdl5n0gaz2v2L5H06QQvpxuyyCOvVt6VEXHpaFcqUvzxsr0zIhb2WxZ55HVbHrv6QEIU\nH0io24q/oU+zyCOvq/K66jE+gDq6bYsPoAKKDyRE8YGEKD6QEMUHEvoPF72a45tCHDcAAAAASUVO\nRK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "x = digits.data\ny = digits.target",
      "execution_count": 4,
      "outputs": []
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "# 效果检验"
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "先来看看使用完全数据集训练的模型能够达到什么样的效果（这里暂不区分训练测试集）"
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "from sklearn.linear_model import LogisticRegression\nclf = LogisticRegression()\nclf.fit(x,y)\nclf.score(x,y)",
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 5,
          "data": {
            "text/plain": "0.993322203672788"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "然而，要想得到完全数据，我们需要标注1797个数字，这是比较吃力的。这还只是一个很小的玩具数据集，对于更大的数据集，数量级数以百万计，我们又怎能标的过来？\n\n那么，我们能不能**只标注一小部分数据**，比如说3%(50个数字左右),让它也达到不错的效果？看看效果如何。"
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "clf2 = LogisticRegression()\nchosen_ids = np.random.choice(range(len(x)),50,replace=False)\nclf2.fit(x[chosen_ids],y[chosen_ids])\nclf2.score(x,y)",
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 6,
          "data": {
            "text/plain": "0.7918753478018921"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "效果下降了很多，这还只是一个简单的任务，对于更难的任务模型的表现还会更差，那么我们就没有办法节省劳动力（偷懒）了吗？\n\n非也，我们还有**主动学习**。"
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "# 主动学习"
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "何谓主动学习？这里我采用一种通俗的讲法：\n\n想象你面对百万大军，要想打败他们未必需要将其全部剿灭，有时只需要斩其**上将**首级即可。\n\n主动学习做的，就是帮助我们找到那个“上将”，解决重点问题，达到事半功倍的效果。看下面的图："
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "![Fig1](https://img-blog.csdnimg.cn/20181213203559446.jpg) "
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "左图中红绿代表两种数据。现在我们只能标注其中有限个数据来训练分类器。中间的图显示的就是随机标注的样本和得到的分界线，准确率大约为70%。而右图就是主动学习方法找到的标注点，因为这些点几乎构成了完美分界线的边界，所以使用与中图同样的样本数，它能够取得90%左右的准确率！"
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "那么我们怎么找到这些关键的点呢？让我们从上例来考虑，上面找到的标注点都在中间区域，实际上都是相对比较**容易混淆的、难以确定类别的**样本。我们的目标就是找到这些样本。**分类器的预测概率**就可以帮助我们衡量这些特性，于是就有了下面的寻找策略："
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "- **RS策略**：随机选择，作为对照。\n- **LC策略**：寻找分类器最没有信心的预测样本，即预测的最可能类别的概率也很低的样本。\n- **BT策略**：寻找分类器最“左右为难”的预测样本，即预测的最可能的两个类别的概率很接近的样本。"
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "下面用代码实验和验证这些策略的效果：\n\n我们进行10批5个样本的标注，对于不同策略选出的样本，使用直接抽取已知标签的方法来模拟手动标注。来看看随着标注的进行，模型的表现如何改善，以及其最终的效果。\n"
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "batch_size = 5\nrounds = 10",
      "execution_count": 7,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "def RS(proba, batch_size):\n    return np.random.choice(range(proba.shape[0]),batch_size,replace=False)\n\ndef LC(proba, batch_size):\n    return np.argsort(np.max(proba,axis=1))[:batch_size]\n\ndef BT(proba, batch_size):\n    sorted_proba = np.sort(proba,axis=1)\n    return np.argsort(sorted_proba[:,-1]-sorted_proba[:,-2])[:batch_size]",
      "execution_count": 8,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "strategies = {\"RS\":RS,\"LC\":LC,\"BT\":BT}\nresults = {\"RS\":[],\"LC\":[],\"BT\":[]}\nfor type0 in strategies:\n    clf = LogisticRegression()\n    anno_batch = RS(x,batch_size)    # 第一批标注样本只能随机选取\n    x_train = x[anno_batch]\n    y_train = y[anno_batch]\n    for i in range(rounds-1):\n        clf.fit(x_train,y_train)\n        prec = clf.score(x,y)\n        results[type0].append(prec)\n        proba = clf.predict_proba(x)\n        stategy0 = strategies[type0]      # 后面采用不同策略\n        anno_batch = stategy0(proba,batch_size)\n        x_train = np.concatenate([x_train,x[anno_batch]])\n        y_train = np.concatenate([y_train,y[anno_batch]])\n        \n    prec = clf.score(x,y)\n    results[type0].append(prec)",
      "execution_count": 9,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "pd.DataFrame(results).plot()\nplt.ylabel(\"Accuracy\")\nplt.xlabel(\"epoches\")\nplt.show()",
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<matplotlib.figure.Figure at 0x1c2728ea048>",
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xd8FNXawPHfyaZ30oEAoROKtBiK\nCgiioAh2QWmKoAIqYn/Fey3XchERvSJSbBQFRFFEFKl2gdBLEnoJkA6kkbZ73j8mxAAhWUI2u0me\nr5/9ZGf2zOyzMcyzc6rSWiOEEEIAONk7ACGEEI5DkoIQQohikhSEEEIUk6QghBCimCQFIYQQxSQp\nCCGEKCZJQQghRDFJCkIIIYpJUhBCCFHM2d4BXK6goCAdERFh7zCEEKJa2bx5c6rWOri8ctUuKURE\nRBATE2PvMIQQolpRSh2xppxUHwkhhCgmSUEIIUQxSQpCCCGKVbs2hdIUFBSQkJBAbm6uvUOpEHd3\nd8LDw3FxcbF3KEKIWq5GJIWEhAR8fHyIiIhAKWXvcC6L1pq0tDQSEhJo3LixvcMRQtRyNaL6KDc3\nl8DAwGqXEACUUgQGBlbbuxwhRM1SI5ICUC0TwjnVOXYhRM1SI6qPhBDC7ixmOLEVDv8G+Tn2jqbC\nJClUEpPJRLt27dBaYzKZ+OCDD/Dx8WHYsGEAHD16FD8/P/z8/AgKCmL16tV2jlgIccVOH4UDa43H\nwfWQe6bohep79y9JoZJ4eHiwbds2AFauXMkLL7zAL7/8Urxv5MiRDBgwgLvuusueYQohrkReFhz+\nvSgRrIG0/cZ+3/oQeSs07Q2Ne4FXoF3DLNUr1iUqSQo2kJGRQZ06dewdhhDiSlkscHJbURJYB8c2\ngKUAnD0g4lq4+iEjEQS1gBrSNljjksIr3+9mz4mMSj1n63q+/PvWNmWWOXv2LB06dCA3N5eTJ0+y\ndu3aSo1BCFFFzhw/v0robLqxP+wq6DbOSAINu4Kzm13DtJUalxTspWT10V9//cXw4cPZtWuX9CwS\nwtHlZ8ORP40ksH8NpMYb+73DoEU/Iwk06QXe5U4wWiPUuKRQ3jf6qtCtWzdSU1NJSUkhJCTE3uEI\nIUqyWCBp5z93A0f/BnM+OLtDo+7QaRg07QMhkTWmSuhy1Lik4Aji4uIwm80EBjpgY5MQtVFmotEm\ncGCN8TMn1dgf2ha6PFxUJdQNXDzsG6cDkKRQSc61KYAxdcXnn3+OyWSyc1RC1FIFZ/+pEjqwDpJ3\nG/u9gqFZn3+qhHzC7BmlQ5KkUEnMZnOZr3/22WdVE4gQtZHWkLT7nyqhI3+COQ9MrsYdwA2vGIkg\ntC04VXwiB7NFk56dT0pmHqlZecU//3meT1p2PmaLpRI/XNWSpCCEqJ7ysyFuRVGV0FrISjL2B0f+\n01W0UXdw9SzzNGaL5lROaRf6i/elZ+dj0Refw83ZiWAfN4K83ajv746LyfFmEFplZTlJCkKI6iXt\nAGz6GLbOh7wz4BEATa83GoebXg++9bCcu9Cn55GamXrehT0lM4+UEhf99Oy8Ui/0rs5OBHu7EeTj\nRngdTzo29CfI26344v/Pc1e83ZwdvqfhR8OsKydJQQjh+Cxm2L8aNs6C/avRTs6kNOjHxqA72Glq\nRWpWISlb80j99QApWXtIz87HXMqVvvhC7+1KfX932of7XXChdzWe+7jhUw0u9LYgSUEI4bByzqRw\n6vdP8Ns9F++cBNKdAlik7uGTnJ6kxNeBeHA1HS2+mNf1c6ddfb/ib/BBPm7F3/aDvN3wda+dF/rL\nIUlBCGF3ZovmSFo28YmZxCZmknNkMx0Sl3BDwS/UVwVssLRiERM4EtSbFvXqMC7Uh5ZhvrQI9SbA\ny1Uu9JVIkoIQokqlZeURl5hpPE5mEJ+Uyd6kTMwF+fR32shw55+JctpLnnJnb90BZLQbSXirq5lS\nxxMnJ7n425okhUri7e1NVlbWRfvnzp3L5MmT0VqjtebBBx/k6aeftkOEQlSt3AIz+5OziD2ZQXxi\nJvFJmcSezCQ1K6+4TJC3K92C83k6fDVXp3+HR14aljpNIPpN3DrcRzsPfzt+gtpJkoIN/fjjj0yb\nNo2ff/6ZevXqkZuby7x58+wdlnB0FktRr5rqMdOuxaJJOHWWuETj4m/cBWRwKDW7uFePm7MTLUJ9\n6NUymFZhPrQK9aGteTf+Oz+F2O9BW6D5jRA9Bqemva9oLIG4MpIUbOjNN99kypQp1KtXDwB3d3dG\njx5t56iEQ8tOhbm3GXPzBDaDBl3+eQS1sPvF8nROPnGJmedd/PcmZpKd/8/gzYYBnrQK8+GWdnVp\nVdeXlmE+RAR6YXJSxnoEOxfD6jnGKGN3f+g2FqJGQUBjO34ycU7NSwo/Pg+JOyv3nGHtoP9bl33Y\nrl276Ny5c+XGImqurBSYOxDSD8F1T0FyHOz9CbYtMF5394cG0UWPrlC/E7h6Vcpbn803k5qVR3IZ\nI3WPnzpLYkZu8TH+ni60DPXh7qgGtAzzMR6hPni5lXJZSTsAm+bA1gXGXVBYOxj4P2h7V7mDy0TV\nsmlSUEr1A94DTMAcrfVbF7zeEPgc8C8q87zWeoUtYxLCIWUlw+e3wqkjcN8iaNLT2K81pB80ZvI8\ntsF47PvZeE2ZjItrw67/JAq/+sWnzC0w/zNQK7P0Ebrnnpf8pl9SHU+X4n783ZsG0iLMx6j+CfMl\n1Net7F4/FjPsW2WMLTiwBpycofVtED3GiFd6DDkkmyUFpZQJmA70BRKATUqpZVrrPSWKTQIWa61n\nKKVaAyuAiCt64wp8o7eVNm3asHnzZnr37m3vUIQjy0w0EsKZBBi6xFjR6xylILCp8eh4P7kFZtJS\nk8g9tAF1bANeyZsJ2PQZLhs+AiDVKZjtTq3YWNCMP/KbEqsbYeb8iRn9z13ovd1oF+5f1I/ftfji\nH1w0WjfQ27Vi0zXkpMPWecao49NHwKcu9Po/6DxCJqCrBmx5pxAN7NdaHwRQSi0EBgElk4IGfIue\n+wEnbBhPlXvhhRd49tlnWb58OWFhYeTl5TFz5kwef/xxe4cmHEXGSfh8gPFz6NecrduFP/YkEZeY\nUVxtc+6bfkpWHpm5hUUHugLXAdfhTCFR7ie41v0gnYinY2EsfdRv4AaFJg8yA9tTUC8a54gueDft\njqtPgG0+y4ltsGk27FwChbnQ6Bro+wq0GgAmF9u8p6h0tkwK9YFjJbYTgC4XlHkZ+Fkp9RjgBdxQ\n2omUUmOAMQANGzas9EArQ05ODuHh4cXbEydOZOLEiSQlJXHDDTegtUYpxYMPPmjHKIVDyTgBnw3A\nkpnIqk4fsmidC3/s/5m8QmOGTV935+LpFyLr+dKjxFw7JeffCfR2xc35gmnazyTAsQ04H91AnWMb\nYPt02Pa+8VpwpFF907Cr0YAd0KTiVTmF+bDnO6OKKGEjuHhC+yEQPRpC7b/glbh8SutSZoKqjBMr\ndTdwk9b6oaLtYUC01vqxEmUmFsXwjlKqG/Ax0FZrfcl5Z6OionRMTMx5+2JjY4mMjLTFx6gyNeEz\nCOtYLJq4+FjqfXc3rrlpDM17ji26BQ0DPOkTGUKfVqFERdTB3aUS1+PIy4ITW4w2iaMbjAt47hnj\nNc+goh5ORYmibgdwcS/7fBknIOZT2PwZZCdDQFNjZtIO90EVjC3QWnMq7xTpZ9NRSmFSJkxOJpyV\nMyYnEyZlwtnJuXi/SRkPJ+VUa0c/K6U2a62jyitnyzuFBKBBie1wLq4eGgX0A9Ba/6WUcgeCgGQb\nxiVElcvJL+S3famsjU1md9xupuf/C5PK5D+Br3Nj+15MjgyhabC37S5Ybt7QuIfxAGMsRGp8UeP1\nRqMhO/4H4zUnF6jX4fzusD6hRqP3kT+Mu4LY5cbYghY3GXcFTSpvbEGBuYCUsykk5ySTmJNIcnYy\nyTnJJOUkFf9MyUkh35JfofOXTBwXJpKSSaRkUrnwmJJlL0xETsoJJ1V9x1nYMilsAporpRoDx4HB\nwH0XlDkK9AE+U0pFAu5Aig1jEqLKHD99lrWxSayJS+bPA2nkF1po4X6KL1z+g6/rWXIHf8sbzbra\nJzgnJ2MN4pBI6DzS2Jed+k8Pp2MbYeNs+OsD47U6EcaCNal7i8YWjIOoBy9rbIHWmqyCrPMv8NnG\nz3P7knKSSM9Nv+hYN5MboZ6hhHiG0D64PaGeoYR6hRLoHggKzBYzZm3GbDFTqAuLtwsthcX7zfry\nylz42rlj8s3555W98PhCSyEa29TAVAWbJQWtdaFSajywEqO76Sda691KqVeBGK31MuApYLZS6kmM\nRueR2lb1WULYmMWi2ZZwmrWxyayOTSIuMROAiEBPhnVtxM3heXRa9xwqLweGLcO1fic7R3wBryBo\ndYvxACjMg5M74FhRd9icU9D9cWh750VjC8wWM+m56cUX9uILfXbSeUkgpzDnorf1d/MnxDOEEM8Q\nWge2Lr74h3oV/fQMxdfVt9ZW+1QWhXW/P5u1KdiKtCkIR5KdV8hv+1JYE5vMuvhkUrPyMTkpOjeq\nww2RIfRuFUrTYC/UqcNGt9O8TBj+nVE9U4Z8cz4WbUFjzJlV8qelqMlNa40Fy0WvX/Tz3DGa4ufF\nr+hSzluiDJri53nmvPMu8MXf7rOTSD2bilmfP9bBWTkT7BlcfMEP9QwtvuCf2w72DMbduZz2C1Ep\nHKFNQYgaKeFUDmtik1kTl8zfB9LIN1vwcXemV8sQbogMoWeLYPw9Xf85IO2AkRAKcmDEMqjb/pLn\ntmgLz/76LCsPr6yCT1JxXi5exRf3LnW7/PPt3jOUEC/jZ4B7QLWuW6+tJCkIUQ6zRbPt2CkjEcQm\nE59kVAs1CfJiRPdG9C7qLVTqQK+0A/DZAKPf/ojvjRHIZfho+0esPLySu1vcTX3v+iilKP5P/fPz\n3MVWYTw/tx84b7vkz+JjlMIJp+L9KM7bLvmeJXvruJpcCfEwEoG3q3dl/XqFg5GkUElMJhPt2rWj\nsLCQxo0bM2/ePPz9/bFYLEyYMIG1a9eilMLd3Z3FixfTuLFM/uXIMnML+G1fanG1UHq2US10dUQd\nJt0SSe9WITQJLufCmLrPSAiWQhi5vNx+++uPrWfG9hkMbDqQl7q+JHXowi4kKVQSDw8Ptm3bBsCI\nESOYPn06L774IosWLeLEiRPs2LEDJycnEhIS8PKqnEnMROU6lp7D6tgk1sYl8/fBNArMGj8PF3q1\nDKZPZCg9WwTj52HlyNyUeKPKSFuMhBBSdnvRkYwjvPDbC0QGREpCEHYlScEGunXrxo4dOwA4efIk\ndevWxamoD3fJUc/C/uITM1m69ThrYpPYl2wsktQ02IsHr2lM71YhdG5UB+fLnf8nOc5ICAAjlkNI\nqzKL5xTk8MTaJ3BxcmHa9dOk4VXYVY1LCv/d+F/i0uMq9ZytAlrxXPRzVpU1m82sWbOGUaNGAXDP\nPfdw7bXX8ttvv9GnTx+GDh1Kx44dKzU+UTG7T5zhzhl/UmjWRDcOYHB0Q/q0CiEi6Aru5JL2GAnB\nydloQwhuUWZxrTWT/pjEoYxDzOw7k3re9Sr+3kJUghqXFOzl7NmzdOjQgcOHD9O5c2f69u0LGHcG\n8fHxrF27lrVr19KnTx+++uor+vTpY+eIa7fUrDzGzN1MHU9Xlo69hjC/Svh2nrTbSAgmV+MOIahZ\nuYd8uvtTVh1ZxcTOE+la104D2YQoocYlBWu/0Ve2c20KZ86cYcCAAUyfPr14NlQ3Nzf69+9P//79\nCQ0N5dtvv5WkYEf5hRbGzt9CalYeXz3SrXISQuJO+HwgOLsbbQiBTcs95K8Tf/Helve4KeImRrYZ\neeUxCFEJpBNxJfPz8+P9999nypQpFBQUsGXLFk6cMKZ8slgs7Nixg0aNGtk5ytrtle93s/FwOpPv\nuoqrwith8raT2407BBdPeOAHqxLC8azjPPvrszTxa8Kr3V+VhmXhMGrcnYIj6NixI+3bt2fhwoUE\nBwczevRo8vLyAIiOjmb8+PF2jrD2mv/3ERZsOMrDPZswqEP98g8oz4mtxprKbj5GG4IVcwHlFuby\n5LonMVvMTLt+Gp4ushylcBySFCpJVlbWedvff/998fN+/fpVdTiiFH8fTOPlZbvp1TKYZ28qu0eQ\nVY5vhnm3g5ufUWVUp/w7QK01r/39GrHpsXzQ+wMa+cpdo3AskhRErZBwKoexC7bQMNCT9wZ3xOR0\nhdU1CTEw7w5j7YCRy8HfusWfFsYvZNmBZYxtP5aeDXpeWQxC2IC0KYgaLye/kNFzN1NQaGH28Cjr\nB6BdyrGNxh2CZx0Y+YPVCWFL0hYmb5xMr/BePNz+4SuLQQgbqTFJobrN9lpSdY7d0WmteearHcQl\nZvD+kI40LW9qivIc3WDcIXgFwcgV4N+g/GOA5JxknvrlKer71OeN696QieKEw6oRf5nu7u6kpaVV\ny4ur1pq0tDTc3WUUqy1MX7efH3ae5Ll+rbi+VciVnezIXzD/DmMVspE/gJ91DdUF5gImrp9IdkE2\n03pNw8fV58riEMKGakSbQnh4OAkJCaSkVM9F29zd3WX6CxtYtSeJKT/vZVCHejzco8mVnezwH7Dg\nbvCtZ7Qh+IRZfehbG99ie8p2pvScQrM65Q9oE8KeakRScHFxkVlHxXn2JmUyYeFW2tX34793XnVl\n4wAO/QZf3AN+DYxupz6hVh+6dN9SFu9dzANtH+CmiJsqHoMQVaRGVB8JUdLpnHxGz43Bw9WZWcM7\n4+5iqvjJDv5i3CH4Nyq6Q7A+IexO3c1//v4PXet25fGOj1c8BiGqkCQFUaMUmi089uVWTpw+y8xh\nnajr51Hxkx1YZ9whBDQx7hC8rW+TSDubxoT1EwjyCGJyj8k4O9WIm3JRC8hfqqhR3vwxjt/2pfLf\nO9vRuVFAxU+0fw0svA8CmxlrKnsFWX1ooaWQZ359hlO5p5jbfy513OtUPA4hqpgkBVFjLNmcwMe/\nH2Jk9wjuvdq6sQOl2rfaSAjBLWDYd+AVeFmHT9s8jU2Jm3j92tdpHdi64nEIYQdSfSRqhK1HT/F/\n3+yke9NAXryl7FXOyrR3JSwcAsEtYfiyy04IPx76kc/3fM6QVkMY2HRgxeMQwk4kKYhqLykjl4fn\nbSbUz43p93XC5XJXSjsn/kdYeD+EtIYRy8Dz8qqf4tPj+fef/6ZjSEeeiXqmYjEIYWeSFES1lltg\nZsy8zWTlFTJ7eBR1vFwrdqK4H2DRMAhrZ7QheFxeO8CZvDNMWDcBbxdvpvaaiovpCqfSEMJObJoU\nlFL9lFLxSqn9SqnnS3n9XaXUtqLHXqXUaVvGI2oWrTX/981Oth87zdR7OtAqzLdiJ4r9HhYPh7rt\nYfi3xiR3l8GiLTz/2/Mk5iQytddUgjysb5QWwtHYrKFZKWUCpgN9gQRgk1JqmdZ6z7kyWusnS5R/\nDJDFi4XVPv79EN9sPc6EG5rTr631I4yLWSywYxEsGw/1OsHQr8H98hPLh9s+5Pfjv/NS15foENLh\n8uMQwoHYsvdRNLBfa30QQCm1EBgE7LlE+SHAv20Yj6hBft2bwhsrYunXJozHeze/vINz0mHbAtj0\nMZw6BA26wtAlxkI5l2nt0bXM3DGT25rdxt0t7r7s44VwNLZMCvWBYyW2E4AupRVUSjUCGgNrbRiP\nqCEOpWYz/osttAj14Z172uNk7doIx7cYiWDXEijMhYbdoPckiBwIzpffFnHozCH+7/f/o01gGyZ1\nnSRLaooawZZJobR/IZeaxnQwsERrbS71REqNAcYANGx4Bf3PRbWXmVvA6LkxmJwUs4dH4eVWzp9w\nwVnYvRQ2zTFWSnPxgg73QdQoCGtb4TiyC7KZsG4Crk6uvNvrXdxMbhU+lxCOxJZJIQEoOdl8OHDi\nEmUHA+MudSKt9SxgFkBUVFT1mx9bVAqLRTNh4TYOpWYzb1Q0DQLKWNs4/RDEfAJb58HZUxDUEvq/\nDe3vBXe/K4pDa81Lf7zE4YzDzO47m7reda/ofEI4ElsmhU1Ac6VUY+A4xoX/vgsLKaVaAnWAv2wY\ni6gB3lkVz5q4ZF4Z2IbuTUvp4WMxw/7Vxl3BvlWgnKDVLRA9GiKug0qq3vl418esOrKKp6OeJrpu\ndKWcUwhHYbOkoLUuVEqNB1YCJuATrfVupdSrQIzWellR0SHAQl0dV8gRVeb77SeYvu4Ag69uwPBu\nFyx2n51m3BHEfAKnj4B3KPR8FjqPNNY/qER/Hv+T/239H/0j+jO89fBKPbcQjkBVt2txVFSUjomJ\nsXcYogrtOn6Guz76k7b1/PhidFdcnZ1Aa6ONYNMc2PUNmPOg0bUQ/RC0GgA2GDyWkJnAvcvvJdQr\nlPn95+PpUkb1lRAORim1WWsdVV45mRBPOLTUrDzGzI2hjqcrM4Z2xtWSC1u+NpLByW3g6g2dhsPV\noyDkCuY8KsfZwrM8uf5JNJr3er0nCUHUWJIUhMPKL7Qwdv4W0rLzWTakHsF/vgpb50PuaQiOhFve\ngavurdD4gsuhtebVv14lPj2eD/p8QAPfBuUfJEQ1JUlBOKxXlu3A9+gqfg3fQOhXv4OTszGm4OqH\noFH3Sms4Ls8XcV+w/OByxnUYR4/wHlXynkLYiyQF4XiyUtj63fs8uvcLwl1T4Ww9uP5Fo5rIpwLT\nWVyBmMQY3t70Nr0a9GLMVWOq9L2FsAdJCsIxaA0Jm2DjbCy7v6WjJZ89Hh0x3zoVU6ubbdJwXJ6k\n7CSe+uUpGvg04I1r38BJyaTCouaTpCDsKz8bdi6BTbMhcScWVx8W6T6s9LyF9x8fjMndPlNQ55vz\nmbh+IrmFuXxy0yf4uNq23UIIRyFJQdhH6n6jB9G2LyDvDIS2Ja//VAb/1YD9pzXfPXANvnZKCABv\nbnyTHak7mNprKk39m9otDiGqmiQFUXXMhbD3J+Ou4OB6cHKB1oMgejQ6PJqJX25je9JJPhl5NU2C\nve0W5td7v2bJ3iWMajuKvo362i0OIexBkoKoGumHYN7txlTVvuHG7KSdRoB3CAAfrNnHDztP8n83\nt6JXyxC7hbkzZSevb3idbnW78VjHx+wWhxD2IklB2F7GSZg7CPIy4d750KI/mP750/t5dyLvrNrL\n7R3rM/q6JnYLM+1sGk+uf5IQzxAm95iMyclkt1iEsBdJCsK2ctJh3m2QkwYjlkH9zue9vDcpkycX\nbeOqcD/evKOd3dYkKLAU8PQvT3M67zTz+s/D3/3yluQUoqaQpCBsJy8T5t9pVB0N/fqihHA6J5/R\nc2PwcHVm5rDOuLvY75v51JipxCTF8Ma1bxAZaLvpMoRwdJIUhG0U5MKXQ+Dkdhi8ABpfd97LhWYL\n47/YysnTuXw5pit1/TzsFCj8cPAH5sfO5/7I+7m16a12i0MIRyBJQVQ+cwEseQAO/w53zIKW/S8q\n8saKOH7fn8rku66ic6M6dgjSEJ8ez8t/vkynkE48FfWU3eIQwlFIUhCVy2KB78ZB/Aq4eQpcdc9F\nRb6KOcYnfxxiZPcI7omyz+RyhZZCvt77NdO3TcfX1Zd3er2Di5P9xkUI4SgkKYjKozX89BzsWGR0\nOY0efVGRLUdP8eLSXVzTLJBJt1R93b3WmvXH1vPulnc5dOYQnUM7M6nLJII8SlnJTYhaSJKCqDzr\nXoeNs6D7Y3Dd0xe9nHgml4fnbSbMz50PhnTC2VS1cwntTt3NlJgpxCTFEOEbwfvXv0+vBr3s1uNJ\nCEdUblIoWlJzgdb6VBXEI6qrPz+AX982ZjLt+9pF01rnFph5eP5mcvIKmT+qC3W8XKsstONZx3l/\ny/usOLSCAPcAJnWZxB0t7pDqIiFKYc2dQhiwSSm1BfgEWCnrKYvzbJkLP78IrW+DAdMuSghaa15c\nuovtx04zc1hnWoZVzeRyGfkZzNkxh/mx83FSToxuN5oH2z6It6v9ptAQwtGVmxS01pOUUi8BNwIP\nAB8opRYDH2utD9g6QOHgdn8L3z8BTfvAHbOhlFHAn/95mK+3JPBEn+bc1Mb26yEUmAtYFL+Ij3Z8\nREZeBgObDmR8x/GEeVXtWgxCVEdWtSlorbVSKhFIBAqBOsASpdQqrfWztgxQOLD9q+HrhyA8Gu6d\nB84XVwn9dSCN136IpW/rUJ7o09ym4WitWXVkFdO2TONY5jG61u3K01FP0zKgpU3fV4iaxJo2hceB\nEUAqMAd4RmtdoJRyAvYBkhRqo6MbYNEwCGkF9y0CV6+LiiScymHcF1toHOTF1Hva4+Rkuwbdbcnb\nmBIzhe0p22nm34wZN8zgmnrXSCOyEJfJmjuFIOAOrfWRkju11hal1ADbhCUcWuJOWHA3+NSFod+A\nx8XzBJ3NNzNm7mYKzBZmDeuMj43WRjiacZRpW6ax6sgqgj2CeaX7KwxqOkgmsxOigqxJCiuA9HMb\nSikfoLXWeoPWOtZmkQnHlLrfmALbzQeGf1c89XVJWmue+3oHsYkZfDLCNmsjnMo9xcwdM1kUtwgX\nkwtjO4xlROsReLp4Vvp7CVGbWJMUZgCdSmxnl7JP1AZnEowZT7WG4d+Cf+mjkWf/dpBl20/wzE0t\nub5V5a6NkGfOY0HsAubsmEN2YTZ3NL+DcR3GyeAzISqJNUlBleyCWlRtZFUDtVKqH/AeYALmaK3f\nKqXMPcDLgAa2a63vs+bcooplp8Lc2yD3DIxcDkGlNxr/ujeFt36M4+Z2YYztVXnLWFq0hRWHVvD+\nlvc5mX2SnuE9ebLzk7JUphCVzJqL+8GixuYZRdtjgYPlHaSUMgHTgb5AAsZYh2Va6z0lyjQHXgCu\n0VqfUkrZb8ktcWm5Z4wqozPHYNhSqNu+1GJH0rJ57MuttAj14e272ldaI+/GkxuZEjOF2PRYIgMi\nee2a1+hSt0ulnFsIcT5rksIjwPvAJIxv82uAMVYcFw3s11ofBFBKLQQGAXtKlBkNTD83WlprnWx9\n6KJK5OfAF4MheQ8MWQiNupdaLDuvkDFzNwMwa1gUXm5XPoPKwdMHmbp5Kr8k/EKYVxhvXPsGtzS5\nBSdVtdNjCFGbWDN4LRkYXIFz1weOldhOAC78etcCQCn1B0YV08ta658q8F7CFgrzYfFwOPoX3PUx\nNC99EXutNU9/tZ19yZl8/mAZ6uE6AAAgAElEQVQ0DQOvrLE39WwqH277kG/2fYOHswcTOk3g/sj7\ncXd2v6LzCiHKZ804BXdgFNAGKP5XqbV+sLxDS9l34fQYzkBzoBcQDvymlGqrtT59QQxjKLo7adiw\nYXkhi8pgMcPSh2H/Krj1PWh75yWLfrj+AD/uSuTFmyO5rnlwhd8ypyCHuXvm8umuT8k353Nvy3t5\npP0j1HG333oLQtQ21tzjzwPigJuAV4H7AWu6oiYAJbunhAMnSinzt9a6ADiklIrHSBKbShbSWs8C\nZgFERUXJvEu2pjX8MBF2fwM3vAKdR16y6Nq4JKb8HM+gDvV46LrGFXo7s8XMsgPL+GDrBySfTeaG\nhjcwofMEGvk2quAHEEJUlDVJoZnW+m6l1CCt9edKqS+AlVYctwlorpRqDBzHqIK6sGfRt8AQ4DOl\nVBBGdVK5jdjCxla/DJs/g2snwrUTLlnsQEoWT3y5jdZ1fXnrjqsq1LD8x/E/eGfzO+w7tY+rgq5i\nSq8pdAzpWPHYhRBXxJqkUFD087RSqi3G/EcR5R2ktS4smnZ7JUZ7wSda691KqVeBGK31sqLXblRK\n7QHMGFNopFXgc4jK8ttU+GMaRI2CPv+6ZLGM3AJGz43B1dmJWcOj8HC9vBHE8enxTN08lT9P/Em4\ndzhTek7hxkY3yrQUQtiZNUlhllKqDkbvo2WAN/CSNSfXWq/AGBFdct+/SjzXwMSih7C3TR/Dmleg\n7V3GUpqXuEBbLJqJi7ZxJC2HBQ91ob6/h9VvkZSdxAfbPuC7/d/h4+rDs1c/y70t78XVVHXrKwgh\nLq3MpFA06V1GUZfRX4EmVRKVqHo7l8APT0Hzm+D2j8Dp0t0+p63Zx+rYZF4Z2IauTQKtOn2hpZCZ\nO2by2a7PMGszI9qM4KF2D+Hn5ldZn0AIUQnKTApFo5fHA4urKB5hD3tXGj2NGl0D93wOpktPXvfT\nrkTeX7OPuzuHM7ybdQ3BOQU5PPvrs/yS8Av9I/rzeKfHCfcJr6zohRCVyJrqo1VKqaeBRRjzHgGg\ntU6/9CGi2jj8uzEWIbQtDPkSXC5dFbQ3KZOnFm+jfQN/XrutrVX1/2ln03hs7WPsTtvNS11f4p6W\n91Rm9EKISmZNUjg3HmFciX0aqUqq/k5sNUYr+zc0psB2971k0TM5BYyZG4OHqzMzh3bG3aX8huWj\nGUd5ZPUjpOSk8G6vd+ndsHdlRi+EsAFrRjRXrPO5cGwp8TD/TvCoA8O+Ba9Ltw2YLZrHF27l+Omz\nfDm6K2F+5Y8s3pW6i3FrxmHRFmbfOJsOIR0qM3ohhI1YM6J5eGn7tdZzKz8cUSVOHTFmPFUmYwps\nv/plFp/yczy/7E3hjdvbERURUO7pf034lad/eZoA9wA+uuEjIvwiKilwIYStWVN9dHWJ5+5AH2AL\nIEmhOspMMtZEKMiGkSsgsOypp5fvOMGM9Qe4r0tD7utS/hQjX+/9mtf+fo2WAS2Z3me6rHMgRDVj\nTfXRYyW3lVJ+GFNfiOrm7CmYfwdkJhqrpoW1LbP4nhMZPPPVDqIa1eHlW9uUWVZrzYztM5ixfQbX\n1L+GqT2nyipoQlRDFZnfOAdjfiJRneRnw4J7IHUv3LcIGkSXWTw9O58x82Lw9XDmw6GdcHW+9LiF\nAksBr/31Gkv3L+W2Zrfxr27/wsXJNmsyCyFsy5o2he/5Z3ZTJ6A1Mm6heinMg4X3w/EYuPtzaFp2\nL6BCs4XxX2whOSOPxY90I8Tn0g3LOQU5PPXLU/x+/Hceaf8IY9uPlakqhKjGrLlTmFLieSFwRGud\nYKN4RGUzF8LXD8HBdTBoOrQeWO4hb/4Yx58H0nj7rqvo0MD/kuVSz6Yybs044tLj+He3f3NXi7sq\nM3IhhB1YkxSOAie11rkASikPpVSE1vqwTSMTV85ige+fgNhlcNOb0HFouYd8syWBj38/xMjuEdwd\n1eCS5Q6fOcyjqx8lLTeN969/n54NelZm5EIIO7FmXcOvAEuJbXPRPuHItIafJ8G2+dDzOeg2ttxD\ndiSc5vlvdtK1SQAv3hJ5yXLbU7Yz7MdhZBdk8/GNH0tCEKIGsSYpOGut889tFD2XKS0d3a9vw9/T\nIfph6PVCucVTMvN4eN5mgr3dmH5fJ1xMpf9prDu6jodWPoSPqw/zb55Pu+B2lR25EMKOrEkKKUqp\n4opopdQgINV2IYkrtusbWPc6tB8C/d665BTY5xSYLYxbsIVTOfnMHNaZQG+3Usstjl/MhPUTaObf\njHn959HQV5ZGFaKmsaZN4RFggVLqg6LtBKDUUc7CAWQmGUtp1usEA/9X5hTY57y2fA8bD6fz3uAO\ntK1/8VTWWmv+t/V/zN45mx7hPXi7x9syBkGIGsqawWsHgK5KKW9Aaa0zbR+WqBCtYfkEyM8x1kQo\nYwrscxZtOsrcv44wpkcTBnW4eLqLAksBL//5MssOLOPO5ncyqesknJ0qMrxFCFEdlPs1Uin1hlLK\nX2udpbXOVErVUUr9pyqCE5dp+0KIXwF9XoLgluUW33L0FC99u5vrmgfx7E0Xl88uyGb8mvEsO7CM\nsR3G8u9u/5aEIEQNZ02bQn+t9elzG0WrsN1su5BEhWScgB+fgwZdoWv5PY2SM3J5ZN5mwvzc+d+Q\njjhf0LCcejaVB356gA0nN/Bq91d5tP2jMihNiFrAmq99JqWUm9Y6D4xxCkDpLZHCPrSGZY+BOR9u\n+xCcyl7rIK/QzMPzN5OZW8jcUdH4e57fmezQmUM8uvpR0nPT+V/v/3Fd+HW2jF4I4UCsSQrzgTVK\nqU+Lth8APrddSOKybZkL+1dD/8nlznqqteZf3+5m69HTfHh/J1qFnb+wzrbkbYxfOx6TMvHpTZ/S\nJqjsifCEEDWLNQ3Nk5VSO4AbAAX8BFi3OK+wvdNHYeWLEHEdXD263OLzNxxlUcwxxl3flJvb1T3v\ntTVH1vDcb88R5hXGjBtm0MDn0iOahRA1kzVtCgCJGKOa78RYTyHWZhEJ61ks8N04QBvzGpXT/XTj\noXReWbab61sGM7Hv+Q3LX8Z9yZPrn6RlnZbM6z9PEoIQtdQl7xSUUi2AwcAQIA1YhNEl9foqik2U\nJ+ZjOPQrDJgGdcq+eTtx+ixjF2ymYYAn0wZ3xORkNBpbtIX3trzHJ7s+oVeDXkzuMRkPZ4+qiF4I\n4YDKqj6KA34DbtVa7wdQSj1ZJVGJ8qUdgFX/gqZ9oPPIMovmFph5eN5mcgssLBzTGT8PY/xCgbmA\nf/35L5YfXM49Le7hhS4vSJdTIWq5suob7sSoNlqnlJqtlOqD0aZgNaVUP6VUvFJqv1Lq+VJeH6mU\nSlFKbSt6PHR54ddSFrNRbeTkYoxaLqOrqNaa/1u6k53Hz/DuvR1oFuIDQFZ+FmPXjGX5weU83vFx\nGZQmhADKuFPQWi8FliqlvIDbgCeBUKXUDGCp1vrnsk6slDIB04G+GFNjbFJKLdNa77mg6CKt9fgr\n+RC1zt8z4OhfcNsM8Lt4FHJJn/5xmG+2HOfJG1rQt3UoAMk5yYxdPZYDpw/wn2v+w6Bmg6oiaiFE\nNVBuQ7PWOltrvUBrPQAIB7YBF33rL0U0sF9rfbBoZtWFgFx9rlTKXlj7GrTob0x4V4Y/96fy+opY\nbmwdymO9mwFw4PQBhq4YyrHMY0zvM10SghDiPNb2PgJAa52utZ6ptS57PUdDfeBYie2Eon0XulMp\ntUMptUQpJV1eymIuhG8fBRcPuPW9MquNjqXnMO6LLTQJ8mLqvR1wclJsTtrMsB+HUWAp4LN+n9G9\nfvcqDF4IUR1cVlK4TKVdsfQF298DEVrrq4DVXGJQnFJqjFIqRikVk5KSUslhViN/vm+ss3zzFPAJ\nvWSxnPxCxszbTKFFM2t4FN5uzqw6sooxP48h0D2Q+TfPJzLw0ovoCCFqL1smhQSg5Df/cOBEyQJa\n67Rz02cAs4HOpZ1Iaz1Lax2ltY4KDg62SbAOL2kPrH8TIgdC2zsvWcxi0UxctJ24xAzeH9KRxkFe\nLIhdwFPrn6J1YGvm9Z9Hfe+y2yGEELWXLZPCJqC5UqqxUsoVY8zDspIFlFIlh9QORAbFlc5cAEsf\nBjdfGPBumdVGU36O56fdibx4cyQ9WwTxTsw7vLXxLXo37M3sG2fj7+5fhYELIaobm/VB1FoXKqXG\nAysBE/CJ1nq3UupVIEZrvQx4vGhVt0IgHRhpq3iqtd/egcQdcM888Aq6ZLElmxP4cP0BhkQ3ZFi3\n+jz/6/P8ePhHBrcczPPRz2MqZ6I8IYRQWl9Yze/YoqKidExMjL3DqDontsGcPtDmdrhzziWLbTyU\nzv1z/ubqiADeGdKESX+8wMbEjUzoNIEH2z4o014LUcsppTZrraPKKyejlRxZYZ7R28gzyJgB9RKO\npGXz8LwYwuu4cf3Vcdyx7EnyzHm8ed2bDGgyoAoDFkJUd5IUHNn6tyB5D9y3GDwDSi1y5mwBoz6P\nwex6AK8mK3l/+wGuqX8NL0S/QCNfmcxWCHF5JCk4qoTN8Mc06DAUWtxUapFCs4WHv1jLcZcFOAdu\npcBSj2nXT6N3g95SXSSEqBBJCo6o4Cx8+wj41IN+b5RexFLA8CVT2aW+ws3PwkPtHmZUu1Eyw6kQ\n4opIUnBEa/8DqXth2FJw97vo5U2Jm3hm3cuk5R+lvkdHZt/yHxr6NrRDoEKImkaSgqM58hf8NR06\nPwBNz59NJCk7iXdi3uHHwz9iya9DK9fHWXTPKJxNthxuIoSoTSQpOJL8bKO3kX8DuPG14t0FlgIW\n7FnAjO0zKLAUok/1pYG6hbnDe0pCEEJUKkkKjmT1y3DqEIxYDm7GugcbTm7gjQ1vcPDMQbqFXcfO\nnb0w5QXwybjueLnJ/z4hROWSq4qjOPgLbJwFXR6BxteRmJ3IlJgprDy8knDvcKb2fJ+PfnQj/fQZ\nFj8cRT1/aVAWQlQ+SQqOIDcDvhsPAU0p6PV/zN35MTN3zMSiLYztMJYH2jzAC1/HsfnIcabf14n2\nDWT+IiGEbUhScAQ/T4KMBP4c9A5v/jSUwxmHub7B9Tx79bOE+4Tzwdp9LN16nKf6tuCWq+qWfz4h\nhKggSQr2tm81idvnM7lVNKu2v0sDnwZM7zOdHuE9AFix8yRTft7LbR3qMb5o9TQhhLAVSQp2lJ+V\nzNxVjzOrQX10QRrjO4xnZNuRuJncANh+7DQTF2+jc6M6vHXnVTJKWQhhc5IU7OSP43/w5toJHPEy\n0Tu4E8/2eOO8xW9OnjnL6LkxBHm7MXNYZ9xdZNprIYTtSVKoYiezTjJ502RWH11Nw4ICZtTtzbU3\nf3hemey8QkZ9FkNOvpl5o7oQ5O1mp2iFELWNJIUqkm/O57PdnzF7x2xA83hWASOcAnC9adp55SwW\nzYRF24hLzODjkVfTMszHPgELIWolSQpV4LeE33hr41sczTxK30Z9eSY5ibppP8HoxeDsel7Z/66M\nY9WeJF6+tTXXtwyxU8RCiNpKkoINHc86zn83/pd1x9YR4RvBzBtm0v10MqwfAde/CHWvOq/84phj\nzPzlIMO6NmJE9wj7BC2EqNUkKdhAnjmPT3d9ypydc3BSTjzR6QmGtx6O69kzsGAo1O0A1z553jF/\nH0zjxaU7ua55EP++tbX0NBJC2IUkhUr2a8KvvLXxLY5lHuPGRjfyzNXPEOYVBlrD8gmQlwm3fwQm\nl+JjDqdm88j8zTQM8OSD+zrJJHdCCLuRpFBJjmUeY/LGyaxPWE+EbwSz+s6iW71u/xTY+RXELYcb\nXoGQyOLdZ3IKePDzTSjgk5FX4+fhcvHJhRCiikhSuEIpOSks2buEOTvnYHIy8WTnJxkWOQyXEncC\nZJyEFc9AeDR0f6x4d4HZwtgvNnMsPYcFD3WlUaCXHT6BEEL8Q5KClbTWJOUksSdtD3vS9hCbHsue\ntD2knk0FoF9EP56KesqoKjr/QPj+CSjMg9tmgJOp+Hz/XrabP/anMeXu9kQ3DqjqjySEEBeRpFAK\nrTUns08WJ4A96XuITYslPTcdACflRBO/JnSv153Wga3pGNKR1oGtSz/ZtgWwbyX0ewuC/pm76NM/\nDvPFhqM82qspd3UOr4qPJYQQ5ar1SUFrTUJWwj93AGmxxKbHcjrvNAAmZaKpf1N6hPcgMiCS1oGt\naRnQEg9nK9YzOJMAP70Aja6B6IeLd6+LS+Y/P+zhpjahPHNjS1t9NCGEuGw2TQpKqX7Ae4AJmKO1\nfusS5e4CvgKu1lrH2Coei7ZwLPNY8cX/3F1AZn4mAM5OzjT3b06fhn2KE0DzOs1xd3a//DfT2lgj\nwWKGQdPByehRFJeYwWNfbqV1PV/evbcDTk7S9VQI4ThslhSUUiZgOtAXSAA2KaWWaa33XFDOB3gc\n2FCZ72/RFg5nHD4vAcSlx5FVkAWAi5MLLeq04KaIm2gd2NpIAP7NcTW5lnNmK8V8AgfXwS3vQEBj\nAFIy8xj1WQxebibmDL8aT9daf6MmhHAwtrwqRQP7tdYHAZRSC4FBwJ4Lyr0GTAaerugbmS1mDp05\nVFz3fy4B5BTmAOBmcqNlnZbc0uSW4gTQ1K/p+T2EKlP6Ifj5JWjSC6JGAZBbYGbMvBjSsvP46uHu\nhPlV4O5DCCFszJZJoT5wrMR2AtClZAGlVEeggdZ6uVLKqqSg0cSnxxf3/tmTtoe9p/ZytvAsAO4m\nd1oGtGRQs0G0DmxNZEAkTfyb4OJURf3/LRaj2sjJBAM/AKXQWvPskh1sPXqaj4Z2ol24X9XEIoQQ\nl8mWSaG0ynJd/KJSTsC7wMhyT6TUGGAMgEeEB3d9fxcAHs4eRAZEcmfzO4kMjKR1QGsa+zXG5GTH\ntQc2zoQjvxsJwb8BAP9bu59l20/wbL+W9Gsry2kKIRyXLZNCAtCgxHY4cKLEtg/QFlhfNM9PGLBM\nKTXwwsZmrfUsYBZAeGS4fvO6N2kd2JpGPo3smwAulLofVr8CzW+EjkMB+H77Caau2sudncJ5tGdT\nOwcohBBls2VS2AQ0V0o1Bo4Dg4H7zr2otT4DBJ3bVkqtB54ur/dRmFcYA5oMsEnAV8Rihm8fBWc3\nuPV9UIqtR0/x9FfbiY4I4I072sokd0IIh2ezmde01oXAeGAlEAss1lrvVkq9qpQaaKv3tZu/PoCE\njXDz2+Bbl+OnzzJ67mZCfd35aFhn3Jwd6I5GCCEuwaZ9IrXWK4AVF+z71yXK9rJlLDaVHAdrX4dW\nA6Dd3WTlFTLqs03kFZpZOKYLAV6V1M1VCCFsTDrKXylzIXz7CLh6wYB3MWuYsHAr+5Kz+HTk1TQL\nkeU0hRDVhySFK2EuhB+ehBNb4e7PwDuEt37Yw+rYZF4b1IYeLYLtHaEQQlwWSQoVlZ8DX4+C+BVw\n3dPQ5nYWbjzK7N8OMbJ7BMO6Rdg7QiGEuGySFCoiJx2+uBcSNsHNUyB6NH8eSGXSt7vo2SKYSbdE\nln8OIYRwQJIULtepIzD/Tjh9FO6dB5G3cjAli0fnb6FJsBf/u6+jLKcphKi2JClcjpPbYcHdxoI5\nw7+DRt1IPJPLqM9jMDkpPh5xNb7uspymEKL6kqRgrQPrYNEwcPcjd9hSVqb4882ajfy2LwVnkxNf\nPNSFBgGe9o5SCCGuiCQFa+xYjP72UXJ8m/FOyOssmnmM7PzD1Pf3YGyvZtzVOZyIIFlfWQhR/UlS\nKIvWpK6cTNDfb7BFtWVk4gQ4ZWbAVfW4vVN9oiMCZJEcIUSNIkmhFKlZeSzfdoyA319hYO4yvjd3\nY1njSbzRuQl9W4fi7iJTVgghaiZJCkVyC8ysjk3imy3H+WvvcaaYpnOLaSM7Gw6j611vc6uvFWsy\nCyFENVerk4LFotl0OJ2lW4/zw46TZOYV0tzXzMrAaTTM3Ao3vk677uPtHaYQQlSZWpkUDqZksXTr\ncZZuPU7CqbN4upro37Yug1s6EfX7aFTafrjzY2h3l71DFUKIKlVrksKp7HyW7zjB11uOs+3YaZwU\nXNMsiKdvbMmNbULxPLUXFtwFeZkw9Gto0tPeIQshRJWr0Ukhr9DMurhkvtlynHXxyRSYNa3CfHjx\n5kgGdqhHqK+7UfDwH7BwCDh7wAMrIKydfQMXQgg7qXFJQWvNlqOn+GbLcZbvOMmZswUE+7gxsnsE\nt3cMp3U93/MP2P0tfDMa6jSGoUvAv6F9AhdCCAdQY5LC0bQcvtmawNKtxzmSloO7ixP92oRxe6dw\nrmkaWPp8RBtmwo/PQYNoGLIQPAOqPnAhhHAg1TopnMkpYPnOEyzdcpyYI6dQCro3DeSx3s3p1zYM\nb7dLfDyLBda8DH+8Z6yWducccJEup0IIUe2Sgtawak8S32xJYE1sMvlmC81DvHmuXytu61iPun7l\nXNwL8+G7cbBzMUSNMtZUdpLBaEIIAdUwKcQmZjB6bgxB3q7c37Uhd3YKp009X5SyYrqJ3AxYPAwO\nrofeL8F1T4E1xwkhRC1R7ZKCt5szn4yM4rrmwbhczroFmYlGl9OkPTDoQ+h4v+2CFEKIaqraJYWG\nAZ70bhV6eQel7oP5d0B2Gty3GJrfYJvghBCimqt2SeGyHdsEX9wDyglGLof6newdkRBCOKyavW5k\n3Ar4/FZw94OHVklCEEKIctTcpBDzKSy6H0JawahVENDE3hEJIYTDs2lSUEr1U0rFK6X2K6WeL+X1\nR5RSO5VS25RSvyulWl/xm2oN696A5ROgaR8YsRy8g6/4tEIIURvYLCkopUzAdKA/0BoYUspF/wut\ndTutdQdgMjD1it7UXAjLHoNf/gsdh8KQL8HN+4pOKYQQtYkt7xSigf1a64Na63xgITCoZAGtdUaJ\nTS9AV/jd8rONSe22zoMez8LAD8DkUuHTCSFEbWTL3kf1gWMlthOALhcWUkqNAyYCrkDvCr1Tdios\nuBtOboMB70LUgxU6jRBC1Ha2vFMobajwRXcCWuvpWuumwHPApFJPpNQYpVSMUiomJSXl/BfTD8LH\nfSF5D9w7XxKCEEJcAVsmhQSgQYntcOBEGeUXAreV9oLWepbWOkprHRUcXKLR+PgW+PhGOHsKhi+D\nVrdUQthCCFF72TIpbAKaK6UaK6VcgcHAspIFlFLNS2zeAuyz+uz7V8NnA4yFcR78GRpeVDMlhBDi\nMtmsTUFrXaiUGg+sBEzAJ1rr3UqpV4EYrfUyYLxS6gagADgFjLDq5Nu+hGXjITgS7v8KfOva6FMI\nIUTtorSueIcfe4hq1UDHDM6Axj2NNgR33/IPEkKIWk4ptVlrHVVeueo3ojnjBLS7G+5fIglBCCEq\nWfWbEM87BG6fBU7VL58JIYSjq35XVt/6khCEEMJG5OoqhBCimCQFIYQQxSQpCCGEKCZJQQghRDFJ\nCkIIIYpJUhBCCFFMkoIQQohikhSEEEIUq3ZzHymlMoF4e8dxgSAg1d5BXMARYwLHjEtiso7EZD1H\njKul1tqnvELVb5oLiLdmUqeqpJSKkZis44hxSUzWkZis54hxKaVirCkn1UdCCCGKSVIQQghRrDom\nhVn2DqAUEpP1HDEuick6EpP1HDEuq2Kqdg3NQgghbKc63ikIIYSwkWqVFJRS/ZRS8Uqp/Uqp5x0g\nnk+UUslKqV32juUcpVQDpdQ6pVSsUmq3UuoJB4jJXSm1USm1vSimV+wd0zlKKZNSaqtSarm9YzlH\nKXVYKbVTKbXN2h4jtqaU8ldKLVFKxRX9bXWzczwti34/5x4ZSqkJ9oypKK4ni/7GdymlvlRKuTtA\nTE8UxbPbqt+R1rpaPAATcABoArgC24HWdo6pB9AJ2GXv30+JmOoCnYqe+wB7HeD3pADvoucuwAag\nq71/V0XxTAS+AJbbO5YSMR0GguwdxwUxfQ48VPTcFfC3d0wlYjMBiUAjO8dRHzgEeBRtLwZG2jmm\ntsAuwBNjCMJqoHlZx1SnO4VoYL/W+qDWOh9YCAyyZ0Ba61+BdHvGcCGt9Umt9Zai55lALMYfqz1j\n0lrrrKJNl6KH3RuzlFLhwC3AHHvH4siUUr4YX4A+BtBa52utT9s3qvP0AQ5orY/YOxCMC6+HUsoZ\n40J8ws7xRAJ/a61ztNaFwC/A7WUdUJ2SQn3gWIntBOx8sXN0SqkIoCPGN3O7Kqqm2QYkA6u01naP\nCZgGPAtY7B3IBTTws1Jqs1JqjL2Dwbg7TwE+Lapqm6OU8rJ3UCUMBr60dxBa6+PAFOAocBI4o7X+\n2b5RsQvooZQKVEp5AjcDDco6oDolBVXKPrt/23RUSilv4GtggtY6w97xaK3NWusOQDgQrZRqa894\nlFIDgGSt9WZ7xnEJ12itOwH9gXFKqR52jscZo5p0hta6I5AN2L1ND0Ap5QoMBL5ygFjqYNReNAbq\nAV5KqaH2jElrHQv8F1gF/IRR7V5Y1jHVKSkkcH6GC8f+t2YOSSnlgpEQFmitv7F3PCUVVTusB/rZ\nOZRrgIFKqcMYVZG9lVLz7RuSQWt9ouhnMrAUo+rUnhKAhBJ3d0swkoQj6A9s0Von2TsQ4AbgkNY6\nRWtdAHwDdLdzTGitP9Zad9Ja98Co7t5XVvnqlBQ2Ac2VUo2Lvh0MBpbZOSaHo5RSGHW/sVrrqfaO\nB0ApFayU8i967oHxjyfOnjFprV/QWodrrSMw/pbWaq3t+q0OQCnlpZTyOfccuBGjCsButNaJwDGl\nVMuiXX2APXYMqaQhOEDVUZGjQFellGfRv8M+GG16dqWUCin62RC4g3J+X9VmQjz9/+3dTWhUVxjG\n8f+TWEpRsNSKiBQFu6mBGs1SBEG6KoiLFEENpasW/KA7rVQCQpciKIIuFCLGj2qNK1E0SKCLVmka\nDAnUhauAyzSQhVXT18V5czsNOrjodCbM81slh5OTewMz770nc5834qWk/cAdyqcNzkfERDOPSdJl\nYBvwoaQpoD8izjXzmChXwH3AeO7hAxyJiFtNPKbVwICkTsqFyI8R0TIfAW0xq4Ch8p7CEuBSRNxu\n7iEBcAAYzAuyJ8BXTZ49jnkAAAJVSURBVD4eco/8M+DrZh8LQET8Kuk6MErZovmd1niy+SdJK4AX\nwL6ImK432U80m5lZZTFtH5mZWYO5KJiZWcVFwczMKi4KZmZWcVEwM7OKi4JZA0ja1krJq2Zvy0XB\nzMwqLgrW1iTtzV4PY5LOZnDfrKTjkkYlDUtamXO7Jf0i6ZGkocy6QdLHku5lv4hRSetz+WU1PQgG\n8ylXJPVIGsnAuzuSVuf4QUmTuf6VpvxBrO25KFjbkvQJsIsSQNcNzAF7gKWUPJ3NlKjh/vyRC8Ch\niPgUGK8ZHwROR8RGStbN0xzfBHwLbKAkjW7JXKpTQG9E9ADngR9y/mFgU67/TWPO2qy+RRNzYdYA\n24Ee4GFexL9Hifb+G7iacy4CNyQtpzSWGcnxAeBa5hStiYghgIh4BpDrPYiIqfx+DFgH/ElpfHI3\n53TyTxF5RImSuAncbMwpm9XnomDtTMBARHz3r0Hp6IJ59bJgXhfpPu+vmq/nKK83ARMR8bp2lp9T\nmtnsAI5K6srGKGb/G28fWTsbBnprUiQ/kLSW8rrozTm7gZ8jYgaYlrQ1x/uAkexVMSVpZ67xbga1\nvckfwMr5HseS3pHUJakD+Cgi7lMa/7wPLPtPz9bsLfhOwdpWRExK+p7S5ayDTJGkNJHpkvQbMEP5\nvwPAl8CZfNOvTQrtA85KOpZrfFHndz6X1AuczC2pJZQOcI+Bizkm4ESLtby0NuGUVLMFJM1GhK/S\nrS15+8jMzCq+UzAzs4rvFMzMrOKiYGZmFRcFMzOruCiYmVnFRcHMzCouCmZmVnkF1upRDmIVZjcA\nAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "以上只是一次随机实验的结果，受每一批的具体样本影响而有一些博定。不过进行多次实验也可以看到，两种主动学习方法利用同样多的样本，却能够达到接近90%的准确率。确实比RS更胜一筹，我们可以使用这个技术来降低达到一定准确度所需的标注量。妈妈再也不用担心我手动标注数据的辛苦了！"
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "# 体验手动标注"
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "下面我也写了一段真正手动标注数据的代码，有兴趣的也不妨亲身尝试一下，用可以接受的时间量标注，打造出一个表现良好的模型："
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "from IPython import display\ndef human_annotate(batch_ids):\n    ret = []\n    plt.gray()\n    for id0 in batch_ids:\n        fig0 = plt.figure(figsize=(1,1)) \n        plt.matshow(digits.images[id0],fignum=0) \n        plt.show()\n        ret.append(int(input(\"请输入这个样本的标签：\")))\n        display.clear_output(wait=True)      # 动态刷新输出\n    return np.array(ret)",
      "execution_count": 11,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "batch_size = 5\nrounds = 3\nresults = []\n\nclf = LogisticRegression()\nanno_batch = RS(proba,batch_size)\nx_train = x[anno_batch]\ny_train = human_annotate(anno_batch)\nfor i in range(rounds-1):\n    clf.fit(x_train,y_train)\n    prec = clf.score(x,y)\n    print(f\"round {i}, prec:{prec}\")\n    results.append(prec)\n    proba = clf.predict_proba(x)\n    sorted_proba = np.sort(proba,axis=1)\n    anno_batch = BT(proba,batch_size)\n    x_train = np.concatenate([x_train,x[anno_batch]])\n    y_train = np.concatenate([y_train,human_annotate(anno_batch)])\nclf.fit(x_train,y_train)\nprec = clf.score(x,y)\nprint(f\"round {rounds}, prec:{prec}\")\nresults.append(prec)\npd.Series(results).plot()\nplt.show()",
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": "round 3, prec:0.5781858653311074\n",
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<matplotlib.figure.Figure at 0x1c275da6978>",
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD8CAYAAACYebj1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VPW9//HXh0DY92xsIeyQuLAE\nEFTqRtyq1tpWqrUsKm50b+/V7tfe29vl19r21tZSK6BWcWm12M3gbkUkiSCQsIVFCEsIhB2yf35/\nZEzHyDKQSc4keT8fjzyYs8185uQw75z5zpyPuTsiIiIAbYIuQEREYodCQURE6igURESkjkJBRETq\nKBRERKSOQkFEROooFEREpI5CQURE6igURESkTtugC6gvISHB09LSgi5DRKRZycvL2+PuiQ29n5gL\nhbS0NHJzc4MuQ0SkWTGz96NxP3r7SERE6igURESkjkJBRETqKBRERKSOQkFEROooFEREpI5CQURE\n6sTc9xRERCRyxyqqeWNDSdTuT6EgItLMlB6p4OU1xWQXFPPmhhLKKmuidt8KBRGRZmDr3qNkF+wi\nu6CY3C2l1Dj07d6BaeNTmZqezAU/js7jKBRERGKQu5O/4yDZ+bVBsHbXIQBGpnRlzsVDycpIIaNv\nN8wsqo+rUBARiRGV1TXkbC4lu6CY7Pxd7DhQRhuDzLRefPvqUWSlp5Dau1Oj1qBQEBEJ0JHyKt5Y\nX0J2QTEvrynmYFkV7du2YcrwRL4ydTiXjkqmV+f4JqtHoSAi0sRKDpXz8ppiFhcU82bhHiqqaujZ\nqR1T01PIykjmwmEJdIoP5uVZoSAi0gQ27znC4oJdZOcXk7d1H+7Qv2dHPjdxIFkZyWQO7EnbuOC/\nOqZQEBFpBDU1zqrtB2o/MZRfzIbdhwHI6NuNL186nKnpyYzq0zXqA8UNpVAQEYmSiqoa3tm8l+z8\n2reGdh0sI66NMXFQL26aWPvR0f49G3eguKEUCiIiDXCorJLX15eQnV/Mq2t3c6i8io7t4vjY8ESy\nMpK5ZGQSPTo13UBxQykURERO0+6DZSxeU0x2fjFvb9xLRXUNvTvHc9XZfWq/SDYsgQ7t4oIu84wo\nFEREIlC4+zCLC4rJLtjF8q37ARjYuxPTJw8kKyOFsak9iWsTW+MDZ0KhICJyHDU1zoqi/WTn1wbB\nppIjAJzTvztfzxpOVkYKw5K6xNxAcUMpFEREQsqrqlmycS+LC2oHiksOldO2jTFpSG9mTE7jslHJ\n9O3RMegyG5VCQURatQPHKnlt3W6yC4p5fV0Jh8ur6Bwfx0UjksjKSOaiEUl079gu6DKbjEJBRFqd\nnQeO8VJB7aWn3964l6oaJ6FLe645ty9ZGclMGty72Q4UN5RCQURaPHdnw+7DZOfvYnFBMe8VHQBg\ncEJnbrtwMFPTkxkzoAdtWsBAcUMpFESkRaqucZZv3Vd3xdEte48CMHpAD/7jihFkpacwNKlLwFXG\nHoWCiLQYZZXVvFW4h+z8Yl5eW8yewxW0izMmD0ng9imDuWxUMsndOgRdZkyLKBTM7Argl0Ac8LC7\n/6je8hnAT4HtoVm/dveHQ8uqgVWh+Vvd/doo1C0iAsD+oxW8snY3iwuKeX19CUcrqunavi0XjUwi\nKz2Zi0Yk0rVD6xkobqhThoKZxQEPAlOBIiDHzBa5e0G9VZ9y9znHuYtj7j664aWKiNTavv8Yi0Md\nyd7ZXEp1jZPcrT2fHNuPrPQUzhvcm/i2wV9xtDmK5ExhAlDo7psAzGwhcB1QPxRERBqFu7N216Ha\nC82t2cXq7QcBGJbUhTs/Npip6Smc06+7BoqjIJJQ6AdsC5suAiYeZ70bzGwKsB74irt/sE0HM8sF\nqoAfufvzDSlYRFqHquoa8t4PDRQX7GJb6THMYGxqT+67ciRT05MZnKiB4miLJBSOF71eb/oF4El3\nLzezO4EFwCWhZanuvsPMBgOvmNkqd9/4oQcwmw3MBkhNTT2tJyAiLceximre3PDv1pT7jlYS37YN\nFwxN4J6LhnLpqGQSu7YPuswWLZJQKAIGhE33B3aEr+Due8Mmfw/8OGzZjtC/m8zsNWAMsLHe9nOB\nuQCZmZn1A0dEWrDSIxW8vKb2i2RvbiihrLKGbh3acumoZKamJzNleCJd2uuDkk0lkj2dAwwzs0HU\nfrpoGnBT+Apm1sfdd4YmrwXWhOb3BI6GziASgPOBn0SreBFpnraVHq37/kDOllJqHPp078CNmQPI\nykhhwqBetIuB1pSt0SlDwd2rzGwO8CK1H0l9xN3zzex+INfdFwFfNLNrqR03KAVmhDYfBfzOzGqA\nNtSOKWiAWqSVcXfydxysC4K1uw4BMDKlK3MuHkpWRgoZfbu1uCuONkfmHlvv1mRmZnpubm7QZYhI\nA1VW15CzuZTs0BVHt+8/RhuDzIG9yMqofWtoYO/OQZfZYphZnrtnNvR+9EadiETN0Yoq3gi1pnx5\n7W4OHKukfds2XDgskS9dNoxLRybRu4sGimOZQkFEGmTP4fLageL8Yv5VuIfyqhp6dGrHZaOSycpI\n5sJhCXSK10tNc6HflIicti17jtS1psx9fx/u0K9HR26amEpWegrj03rSVgPFzZJCQUROyd1Ztf1A\nXWvK9cWHAUjv040vXTqMrPQURvXpqoHiFkChICLHVVFVwzub95KdX8xLa4rZeaCMuDbGhLRefPfj\nqUxNT2ZAr05BlylRplAQkTqHy6t4fV0J2QW7eGXtbg6VVdGxXRxThifw9awRXDIyiZ6d44MuUxqR\nQkGkldt9qIyXCnaTXbCLJYV7qaiuoVfneK48K4Ws9BQuGJbQaltTtkYKBZFWaGPJ4dqB4vxdLN+2\nH3dI7dWJz08aSFZGCuMG9iROVxxtlRQKIq1ATY3zXtH+um8Ubyw5AsDZ/brz1cuGk5WRwvDkLhoo\nFoWCSEtVXlXN2xv3kl1QzEsFxew+VE7bNsZ5g3szfXIal41Kpm+PjkGXKTFGoSDSghwsq+TVUGvK\n19aVcLi8is7xcVw0Iomp6clcPCKJ7p3UmlJOTKEg0sztOlDG4jW1bwst3bSXymonoUs815zbh6z0\nFCYN6a2BYomYQkGkmXF3CncfDnUkK+a9bfsBGJTQmVkXDCIrPZnRAzRQLGdGoSDSDFTXOMu37gtd\nWqKYzXtqB4pHD+jBNy4fweUZyQxJ1ECxNJxCQSRGlVVWs2TjnrpvFO85XEG7OGPSkARuvWAQU9OT\nSe7WIegypYVRKIjEkANHK3llXXHdQPHRimq6tG/LxSNrB4ovGpFItw4aKJbGo1AQCdiO/cfqrjj6\nzqZSqmqcpK7tuX5MP7IyUjhvcC/at9VAsTQNhYJIE3N31hUfqrvi6OrtBwEYmtSF2VMGk5WRwjn9\nutNGA8USAIWCSBOornFyt/y7NeXW0qOYwdjUntx35UimpiczOLFL0GWKKBREGktZZTVvbthDdv4u\nXl67m9IjFcTHteH8ob2566IhXDoqiaSuGiiW2KJQEImifUcqeHntbrLzd/Hmhj0cq6yma4e2XDoy\niayMFKYMT6RLe/23k9ilo1OkgbaVHq0bKM7Zso/qGqdP9w58JrM/U9NTmDi4F+3UmlKaCYWCyGly\ndwp2HgwNFBezZmftQPGI5K7cfdEQstJTOKtfN32RTJolhYJIhFYVHeDPy4vIzi9m+/5jmMH4gb34\n9tWjmJqezMDenYMuUaTBFAoiEXjina185y+radvGuHBYIl+6dBiXjEoioUv7oEsTiSqFgshJ1NQ4\nP81ex29f28jFIxL55WfH6BvF0qIpFEROoKyymq8/8x5/XbmTmyem8l/XZtBWA8bSwikURI5j35EK\nZj+WS86Wfdx35UhmTxmsgWNpFRQKIvW8v/cIM+blsH3/MX590xg+fk7foEsSaTIKBZEw727dx20L\ncnF3nrhtIplpvYIuSaRJKRREQv6xaidffmoFKd07MH/mBAYl6COm0vpENGpmZleY2TozKzSze4+z\nfIaZlZjZitDPbWHLppvZhtDP9GgWLxIN7s7Db27i7ifeJaNvN/5812QFgrRapzxTMLM44EFgKlAE\n5JjZIncvqLfqU+4+p962vYDvAZmAA3mhbfdFpXqRBqquce5/IZ8Fb7/PVWen8PPPjFaTe2nVIjlT\nmAAUuvsmd68AFgLXRXj/lwOL3b00FASLgSvOrFSR6DpaUcUdj+Wy4O33uWPKYH792bEKBGn1IgmF\nfsC2sOmi0Lz6bjCzlWb2rJkNOJ1tzWy2meWaWW5JSUmEpYucud2Hyrjxd0t5Ze1ufnBdBvddNUpN\nbUSILBSO9z/F602/AKS5+znAS8CC09gWd5/r7pnunpmYmBhBSSJnbn3xIa5/cAmFuw/z+89ncsuk\ntKBLEokZkYRCETAgbLo/sCN8BXff6+7locnfA+Mi3VakKS3ZuIcbfruEiuoanr5jEpeOSg66JJGY\nEkko5ADDzGyQmcUD04BF4SuYWZ+wyWuBNaHbLwJZZtbTzHoCWaF5Ik3uT3lFTH9kGX26d+D5e87n\n7P7dgy5JJOac8tNH7l5lZnOofTGPAx5x93wzux/IdfdFwBfN7FqgCigFZoS2LTWzH1AbLAD3u3tp\nIzwPkRNyd371ciEPvLSeyUN689vPjaN7R13UTuR4zP0jb/EHKjMz03Nzc4MuQ1qIiqoavvncKp7N\nK+KGsf3530+eTXxbXdROWh4zy3P3zIbej77RLC3WwbJK7no8j7cK9/KVy4bzxUuH6qJ2IqegUJAW\nafv+Y8yct4xNJUf42afP5YZx/YMuSaRZUChIi7N6+wFmzs+hrLKaR2dNYPLQhKBLEmk2FArSoryy\ntpg5TyynZ6d4/njbRIYndw26JJFmRaEgLcZjS9/ne39ZTXrfbjwyfTxJ3ToEXZJIs6NQkGavpsb5\n0T/XMveNTVw6MolffXYMndvr0BY5E/qfI81aWWU1X3v6Pf62aie3nDeQ712Trj7KIg2gUJBmq/RI\nBbc/msu7W/fx7atHcesFg/SRU5EGUihIs7R5zxFmzlvGzgNl/OamsVx5dp9TbyQip6RQkGYn7/1S\nbluQi5nxxO3nMW5gz6BLEmkxFArSrPxt5U6+8vQK+vXoyLwZ40lT20yRqFIoSLPg7sx9YxP/+4+1\nZA7sye8/n0nPzvFBlyXS4igUJOZVVdfw/RfyeXzpVq4+pw8/+/S5apsp0kgUChLTjpRXMeeJd3l1\nXQl3fmwI/3H5CLXNFGlECgWJWcUHy5g1P4c1Ow/yP9efxc0TBwZdkkiLp1CQmLRu1yFmzlvG/mOV\n/GH6eC4emRR0SSKtgkJBYs5bhXu487E8OrWP4+k7JnFWP7XNFGkqCgWJKc/kbuO+P69iSGIX5s0c\nT98eHYMuSaRVUShITHB3HnhpA796eQMXDkvgwZvH0q2D+iiLNDWFggSuoqqGe/+0kj8v386nx/Xn\nh588m3a6qJ1IIBQKEqgDxyq587E83t60l69NHc6cS9RHWSRICgUJzLbSo8yan8OWvUd44MZzuX6M\n+iiLBE2hIIFYWbSfWfNzqaiq5tFZE5k0pHfQJYkICgUJwEsFxXzhyeX07hLPwtkTGZqkPsoisUKh\nIE3q0be38P1F+ZzVrzsPT88kqav6KIvEEoWCNImaGueHf1/Dw//azNT0ZH45bTSd4nX4icQa/a+U\nRldWWc1XnlrBP1bvYsbkNL7z8XTidFE7kZikUJBGtfdwObc9msuKbfv5zsfTufWCQUGXJCInoVCQ\nRrOp5DAz5uVQfLCM3948jivOSgm6JBE5BYWCNIqcLaXc/mgucWYsnH0eY1LVR1mkOYjoWgJmdoWZ\nrTOzQjO79yTrfcrM3MwyQ9NpZnbMzFaEfh6KVuESu154bwc3//4denWO57m7z1cgiDQjpzxTMLM4\n4EFgKlAE5JjZIncvqLdeV+CLwDv17mKju4+OUr0Sw9ydh17fxI//uZYJab2Y+/lx9OikPsoizUkk\nZwoTgEJ33+TuFcBC4LrjrPcD4CdAWRTrk2aiqrqGbz63mh//cy3XntuXx26boEAQaYYiCYV+wLaw\n6aLQvDpmNgYY4O5/Pc72g8xsuZm9bmYXnnmpEqsOl1dx64Jcnly2lXsuHsIvbhxN+7ZxQZclImcg\nkoHm432g3OsWmrUBHgBmHGe9nUCqu+81s3HA82aW4e4HP/QAZrOB2QCpqakRli6xYNeB2j7K64oP\n8aNPns20Cfr9iTRnkZwpFAEDwqb7AzvCprsCZwGvmdkW4DxgkZllunu5u+8FcPc8YCMwvP4DuPtc\nd89098zExMQzeybS5NbsPMj1v3mL9/ce4ZEZ4xUIIi1AJKGQAwwzs0FmFg9MAxZ9sNDdD7h7grun\nuXsasBS41t1zzSwxNFCNmQ0GhgGbov4spMm9sb6ETz/0Nu7wzJ2T+dhwhblIS3DKt4/cvcrM5gAv\nAnHAI+6eb2b3A7nuvugkm08B7jezKqAauNPdS6NRuATn6ZxtfPO5VQxNqu2j3Ke7+iiLtBTm7qde\nqwllZmZ6bm5u0GXIcbg7P8tez69fLWTK8EQevGkMXdVHWSQmmFmeu2c29H70jWaJSHlVNf/57Eqe\nX7GDaeMH8INPnKU+yiItkEJBTunA0UpmP5bLO5tL+cblI7j7oiHqoyzSQikU5KS2lR5lxrxlbCs9\nxi+njea60f1OvZGINFsKBTmhFdv2c9uCHCqrncduncDEweqjLNLSKRTkuLLzd/HFhctJ7NqehTMm\nMDSpS9AliUgTUCjIR8x7azP3/7WAc/r34A/TM0no0j7okkSkiSgUpE51jfPffytg3ltbuDwjmV/c\nOIaO8bqGkUhrolAQAI5VVPOlhcvJLihm1vmD+NbVo9RHWaQVUigIew6Xc+uCXFYW7ed716Qz83z1\nURZprRQKrVzh7sPMnL+MkkPl/O5z48jKUB9lkdZModCKvbNpL7Mfy6NdnPHU7EmcO6BH0CWJSMAU\nCq3UX1Zs5xvPrGRAr47MnzmBAb06BV2SiMQAhUIr4+785rWN/PTFdUwc1Iu5t2TSvZMuaicitRQK\nrUhldQ3feX41C3O28YnRffnxp85R20wR+RCFQitxqKySu//4Lm9u2MMXLhnKV6cO10XtROQjFAqt\nwM4Dx5g5L4fC3Yf5yQ3n8JnxA069kYi0SgqFFi5/xwFmzc/hSHk182aO58JhapspIiemUGjBXl9f\nwt2P59GtYzuevWsSI1O6BV2SiMQ4hUIL9eSyrXz7+dWMSO7KvJnjSe7WIeiSRKQZUCi0MDU1zv/L\nXsdvXtvIRSMS+fVNY+nSXr9mEYmMXi1akPKqar7+zEpeeG8HN01M5f5rM2irPsoichoUCi3EviMV\n3PFYHsu2lHLvlSO5Y8pgfeRURE6bQqEF2Lq3to9y0b5j/N9nx3DNuX2DLklEmimFQjP37tZ93L4g\nl2p3/nj7RMan9Qq6JBFpxhQKzdg/V+/iSwuXk9K9A/NmjGdwovooi0jDKBSaIXfnD//azP/8fQ2j\nB/Tg4c9n0lt9lEUkChQKzUx1jfODvxYwf8kWrjwrhQduHE2HdrqonYhEh0KhGTlaUcUXn1zBS2uK\nuf3CQdx35SjaqI+yiESRQqGZ2H2ojNsW5LJ6+wHuvy6Dz09KC7okEWmBFArNQOHuQ0x/JIfSIxXM\nvSWTy9KTgy5JRFoohUKMW7JxD3c+lkf7dnE8fcckzu7fPeiSRKQFi+gaCGZ2hZmtM7NCM7v3JOt9\nyszczDLD5t0X2m6dmV0ejaJbi+eWFzH9kWUkd+vAc3dPViCISKM75ZmCmcUBDwJTgSIgx8wWuXtB\nvfW6Al8E3gmblw5MAzKAvsBLZjbc3auj9xRaHnfn/14p5OeL1zNpcG8eumUc3Tuqj7KINL5IzhQm\nAIXuvsndK4CFwHXHWe8HwE+AsrB51wEL3b3c3TcDhaH7kxOorK7hP55dyc8Xr+eTY/uxYNYEBYKI\nNJlIQqEfsC1suig0r46ZjQEGuPtfT3db+beDZZXMnJfDM3lFfOnSYfzs0+cS31ZXORWRphPJQPPx\nPgjvdQvN2gAPADNOd9uw+5gNzAZITU2NoKSWZ8f+2j7KG0sO8/8+fS6fGtc/6JJEpBWKJBSKgPBO\n7/2BHWHTXYGzgNdCl2pOARaZ2bURbAuAu88F5gJkZmZ+JDRautXba/soH6uoZsGsCZw/NCHokkSk\nlYrkvYkcYJiZDTKzeGoHjhd9sNDdD7h7grunuXsasBS41t1zQ+tNM7P2ZjYIGAYsi/qzaMZeXbub\nz/zubdrFteHZuyYrEEQkUKc8U3D3KjObA7wIxAGPuHu+md0P5Lr7opNsm29mTwMFQBVwjz559G+P\nL32f7/5lNel9u/HI9PEkqY+yiATM3GPr3ZrMzEzPzc0NuoxGVVPj/PjFtfzu9U1cMjKJ//vsGDqr\nj7KINICZ5bl75qnXPDm9EjWxsspqvvbMe/xt5U4+d14q379GfZRFJHYoFJpQ6ZEKZj+aS+77+/jm\nVSO5/UL1URaR2KJQaCJb9hxh5vwctu8/xoM3jeXqc/oEXZKIyEcoFJpA3vv7uP3RXNydJ2+fyLiB\n6qMsIrFJodDI/r5qJ19+agV9u3dg/swJpCV0DrokEZETUig0Enfn4Tc388N/rGFsak9+//lMenWO\nD7osEZGTUig0gqrqGv7rhQIeW/o+V5/dh5995lz1URaRZkGhEGVHyqv4wpPLeWXtbu6YMpj/vGKk\n+iiLSLOhUIii3QfLmLUgh4IdB/nBJ87ilvMGBl2SiMhpUShEyfriQ8ycl8O+oxX8Yfp4Lh6ZFHRJ\nIiKnTaEQBUsK93DH43l0DPVRPquf2maKSPOkUGigZ/OKuPdPKxmc2Jl5MyfQr0fHoEsSETljCoUz\n5O788uUN/OKlDZw/tDe//dw4unVQ20wRad4UCmegoqqG+/68ij+9W8SnxvXnh9efrbaZItIiKBRO\n04Fjldz1eB5LNu7lq1OH84VLhuqidiLSYigUTkPRvqPMnJfDlr1H+PlnzuWTY9VHWURaFoVChFYV\nHWDWghzKKmv7KE8eoraZItLyKBQi8PKaYuY8sZxeneN54raJDEvuGnRJIiKNQqFwCo+9vYXvLcon\no293/jAjk6Su6qMsIi2XQuEEamqcH/1zLXPf2MRlo5L41WfH0Cleu0tEWja9yh1HWWU1X316BX9f\ntYvpkwby3WsyiNNF7USkFVAo1LP3cDm3P5rL8m37+fbVo7j1gkH6yKmItBoKhTCb9xxhxrxl7DpQ\nxm9vHssVZ6mPsoi0LgqFkNwtpdz+aC5mxpOzz2Nsas+gSxIRaXIKBeCF93bwtWfeo3+PjsybOZ6B\nvdVHWURap1YdCu7OQ69v4sf/XMv4tJ7MvSWTnuqjLCKtWKsNharqGr63KJ8/vrOVa87ty08/dY76\nKItIq9cqQ+FweRVznniX19aVcNdFQ/hG1gj1URYRoRWGQvHBMmbNz2HtrkP88PqzuWliatAliYjE\njFYVCmt3HWTmvBwOHqvk4emZXDxCfZRFRMK1mlD414Y93PV4Hp3ax/H0nZPI6Ks+yiIi9UXULszM\nrjCzdWZWaGb3Hmf5nWa2ysxWmNm/zCw9ND/NzI6F5q8ws4ei/QQi8XTuNmbMW0a/nh15/p7zFQgi\nIidwyjMFM4sDHgSmAkVAjpktcveCsNWecPeHQutfC/wcuCK0bKO7j45u2ZFxdx5YvJ5fvVLIhcMS\n+M3NY+mqPsoiIicUydtHE4BCd98EYGYLgeuAulBw94Nh63cGPJpFnonyqmru/dMqnlu+nRszB/Df\n159Fuzj1URYROZlIQqEfsC1sugiYWH8lM7sH+CoQD1wStmiQmS0HDgLfdvc3z7zcyBw4Wskdj+ey\ndFMpX88azj0Xq4+yiEgkIvnT+Xivph85E3D3B919CPCfwLdDs3cCqe4+htrAeMLMun3kAcxmm1mu\nmeWWlJREXv1xbCs9yg0PLeHd9/fzixtHM+eSYQoEEZEIRRIKRcCAsOn+wI6TrL8Q+ASAu5e7+97Q\n7TxgIzC8/gbuPtfdM909MzExMdLaP+K9bfu5/jdvsftgGY/eOoFPjOl3xvclItIaRRIKOcAwMxtk\nZvHANGBR+ApmNixs8mpgQ2h+YmigGjMbDAwDNkWj8PoWFxQzbe5SOrSL4893T+a8wb0b42FERFq0\nU44puHuVmc0BXgTigEfcPd/M7gdy3X0RMMfMLgMqgX3A9NDmU4D7zawKqAbudPfSaD+J+W9t5r/+\nWsA5/brz8PTxJHZtH+2HEBFpFcw98A8KfUhmZqbn5uZGtG51jfPDv6/hD//azNT0ZH41bQwd43VR\nOxFpfcwsz90zG3o/zfYbzccqqvnyU8t5Mb+YGZPT+M7H09VHWUSkgZplKOw5XM5tC3J5r2g/3/14\nOrMuGBR0SSIiLUKzC4WNJYeZMW8ZJYfKeehz47g8IyXokkREWoxmFQrLNtf2UW4XZyycPYnRA3oE\nXZKISIvSbELhLyu2841nVtK/V0fmz5hAau9OQZckItLixHwouDu/eW0jP31xHRMG9WLuLePo0Ul9\nlEVEGkNMh0JldQ3f/ctqnly2jetG9+UnnzqH9m31kVMRkcYSs6FwuLyKu//4Lm+sL2HOxUP5WtZw\nXcNIRKSRxWQo7DxwjFnzc1lffIgfffJspk1QH2URkaYQc6FQVlnN9Q8u4XB5FfNmjGfK8DO/QJ6I\niJyemOs6s7HkCABP3zFJgSAi0sRi7kyhU3wcz99zPindOwRdiohIqxNzZwqDEjorEEREAhJzoSAi\nIsFRKIiISB2FgoiI1FEoiIhIHYWCiIjUUSiIiEgdhYKIiNRRKIiISB1z96Br+BAzOwSsC7qOCCQA\ne4IuIgKqM7pUZ3Q1hzqbQ40AI9y9a0PvJOYucwGsc/fMoIs4FTPLVZ3RozqjS3VGT3OoEWrrjMb9\n6O0jERGpo1AQEZE6sRgKc4MuIEKqM7pUZ3SpzuhpDjVClOqMuYFmEREJTiyeKYiISECaNBTM7Aoz\nW2dmhWZ273GWtzezp0LL3zGztLBl94XmrzOzywOu86tmVmBmK83sZTMbGLas2sxWhH4WBVznDDMr\nCavntrBl081sQ+hnesB1PhAi4P0nAAAE30lEQVRW43oz2x+2rEn2p5k9Yma7zWz1CZabmf0q9BxW\nmtnYsGVNuS9PVefNofpWmtkSMzs3bNkWM1sV2pdR+aRKA+q8yMwOhP1uvxu27KTHSxPW+I2w+laH\njsVeoWVNuS8HmNmrZrbGzPLN7EvHWSd6x6e7N8kPEAdsBAYD8cB7QHq9de4GHgrdngY8FbqdHlq/\nPTAodD9xAdZ5MdApdPuuD+oMTR+Oof05A/j1cbbtBWwK/dszdLtnUHXWW/8LwCMB7M8pwFhg9QmW\nXwX8AzDgPOCdpt6XEdY5+YPHB678oM7Q9BYgIUb250XAXxt6vDRmjfXWvQZ4JaB92QcYG7rdFVh/\nnP/rUTs+m/JMYQJQ6O6b3L0CWAhcV2+d64AFodvPApeamYXmL3T3cnffDBSG7i+QOt39VXc/Gppc\nCvRvpFpOJpL9eSKXA4vdvdTd9wGLgStipM7PAk82Ui0n5O5vAKUnWeU64FGvtRToYWZ9aNp9eco6\n3X1JqA4I7tiMZH+eSEOO69NymjUGclwCuPtOd383dPsQsAboV2+1qB2fTRkK/YBtYdNFfPSJ1a3j\n7lXAAaB3hNs2ZZ3hbqU2oT/QwcxyzWypmX2iMQoMibTOG0Knk8+a2YDT3DYaIn6s0Ntwg4BXwmY3\n1f48lRM9j6bcl6er/rHpQLaZ5ZnZ7IBqCjfJzN4zs3+YWUZoXsztTzPrRO0L6Z/CZgeyL632LfUx\nwDv1FkXt+GzKbzTbcebV/+jTidaJZNtoifixzOxzQCbwsbDZqe6+w8wGA6+Y2Sp33xhQnS8AT7p7\nuZndSe1Z2CURbhstp/NY04Bn3b06bF5T7c9TiYVjM2JmdjG1oXBB2OzzQ/syCVhsZmtDfy0H4V1g\noLsfNrOrgOeBYcTm/rwGeMvdw88qmnxfmlkXaoPpy+5+sP7i42xyRsdnU54pFAEDwqb7AztOtI6Z\ntQW6U3t6F8m2TVknZnYZ8C3gWncv/2C+u+8I/bsJeI3aVA+kTnffG1bb74FxkW7blHWGmUa9U/Qm\n3J+ncqLn0ZT7MiJmdg7wMHCdu+/9YH7YvtwNPEfjvQV7Su5+0N0Ph27/HWhnZgnE4P7k5Mdlk+xL\nM2tHbSD80d3/fJxVond8NsVASWjAoy21gxyD+PcAUka9de7hwwPNT4duZ/DhgeZNNN5AcyR1jqF2\nMGxYvfk9gfah2wnABhpvkCySOvuE3b4eWOr/HnzaHKq3Z+h2r6DqDK03gtrBOwtif4YeI40TD4xe\nzYcH8pY19b6MsM5UasfcJteb3xnoGnZ7CXBFgHWmfPC7pvYFdWto30Z0vDRFjaHlH/xh2jmofRna\nL48CvzjJOlE7PhvtgDhB4VdRO3K+EfhWaN791P61DdABeCZ0UC8DBodt+63QduuAKwOu8yWgGFgR\n+lkUmj8ZWBU6kFcBtwZc5/8C+aF6XgVGhm07K7SfC4GZQdYZmv4+8KN62zXZ/qT2L8GdQCW1f13d\nCtwJ3BlabsCDoeewCsgMaF+eqs6HgX1hx2ZuaP7g0H58L3RMfCvgOueEHZtLCQux4x0vQdQYWmcG\ntR9yCd+uqfflBdS+5bMy7Pd6VWMdn/pGs4iI1NE3mkVEpI5CQURE6igURESkjkJBRETqKBRERKSO\nQkFEROooFEREpI5CQURE6vx/Va9zmmawxaoAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "name": "conda-env-py36-py",
      "display_name": "Python [conda env:py36]",
      "language": "python"
    },
    "toc": {
      "nav_menu": {
        "width": "160px",
        "height": "12px"
      },
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "base_numbering": 1,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": false,
      "toc_position": {},
      "toc_section_display": true,
      "toc_window_display": false
    },
    "language_info": {
      "name": "python",
      "version": "3.6.2",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "gist": {
      "id": "d48c41ebbc00fffa05ae498632187bd6",
      "data": {
        "description": "active learning1.ipynb",
        "public": true
      }
    },
    "_draft": {
      "nbviewer_url": "https://gist.github.com/d48c41ebbc00fffa05ae498632187bd6"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}