Why does BN shows bad performance? - V2

BN2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets('MNIST_data/', one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weight_init(shape):\n",
    "    return tf.truncated_normal(shape, stddev=0.1)\n",
    "\n",
    "def bias_init(shape):\n",
    "    return tf.constant(0.1, shape=shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 참조한 image completion 코드에서는 다른 식으로 구현하는데, 그게 더 빠른가?\n",
    "# 특이하게 구현함. https://github.com/bamos/dcgan-completion.tensorflow/blob/master/ops.py\n",
    "def lrelu(x, leak=0.2):\n",
    "    return tf.maximum(x, x*leak)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def batch_norm(summed_input, is_training):\n",
    "#     return summed_input # without BN\n",
    "    return tf.layers.batch_normalization(summed_input, axis=-1, training=is_training)\n",
    "#     return tf.contrib.layers.batch_norm(summed_input, center=True, scale=True, is_training=is_training)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.float32, shape=[None, 784])\n",
    "Y = tf.placeholder(tf.float32, shape=[None, 10])\n",
    "isTraining = tf.placeholder(tf.bool)\n",
    "\n",
    "X_img = tf.reshape(X, [-1, 28, 28, 1])\n",
    "\n",
    "W1 = tf.Variable(weight_init([5, 5, 1, 64]))\n",
    "a1 = tf.nn.conv2d(X_img, W1, strides=[1, 2, 2, 1], padding='SAME')\n",
    "bn1 = batch_norm(a1, isTraining)\n",
    "h1 = lrelu(bn1)\n",
    "\n",
    "W2 = tf.Variable(weight_init([5, 5, 64, 128]))\n",
    "a2 = tf.nn.conv2d(h1, W2, strides=[1, 2, 2, 1], padding='SAME')\n",
    "bn2 = batch_norm(a2, isTraining)\n",
    "h2 = lrelu(bn2)\n",
    "\n",
    "W3 = tf.Variable(weight_init([6272, 10]))\n",
    "b3 = tf.Variable(bias_init([10]))\n",
    "h2_flat = tf.reshape(h2, [-1, 6272])\n",
    "\n",
    "# last layer activation = logit\n",
    "logits = tf.matmul(h2_flat, W3) + b3\n",
    "y_prob = tf.nn.softmax(logits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=Y))\n",
    "solver = tf.train.AdamOptimizer().minimize(loss)\n",
    "\n",
    "pred = tf.argmax(logits, axis=1)\n",
    "correction = tf.equal(pred, tf.argmax(Y, axis=1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correction, \"float\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def train():\n",
    "    # moving average 때문에 얘를 해줘야 된다는데 잘 모르겠음\n",
    "    update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "    with tf.control_dependencies(update_ops):\n",
    "        solver = tf.train.AdamOptimizer().minimize(loss)\n",
    "    \n",
    "    batch_size = 100\n",
    "    total_batch = mnist.train.num_examples / batch_size\n",
    "    history = []\n",
    "\n",
    "    sess = tf.Session()\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    for epoch in range(20):\n",
    "        loss_sum = 0\n",
    "        for i in range(total_batch / 20):\n",
    "            batch = mnist.train.next_batch(batch_size)\n",
    "\n",
    "            loss_cur, _ = sess.run([loss, solver], feed_dict={X: batch[0], Y: batch[1], isTraining: True})\n",
    "            loss_sum += loss_cur\n",
    "\n",
    "        # accuracy & loss calc\n",
    "        train_loss = loss_sum / total_batch\n",
    "#         test_loss = sess.run(loss, feed_dict={X: mnist.test.images, Y: mnist.test.labels, isTraining: False})\n",
    "\n",
    "        # calculate accuracy for both train and test\n",
    "        train_acc = sess.run(accuracy, {X: mnist.train.images[:10000], Y: mnist.train.labels[:10000], isTraining: False})\n",
    "        test_loss, test_acc = sess.run([loss, accuracy], {X: mnist.test.images, Y: mnist.test.labels, isTraining: False})\n",
    "        print(\"[{:3}] train: {:.5f} / test: {:.5f} | [acc] train: {:.4f} / test: {:.4f}\"\n",
    "              .format(epoch+1, train_loss, test_loss, train_acc, test_acc))\n",
    "        history.append([train_acc, test_acc])\n",
    "    \n",
    "    return history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  1] train: 0.09566 / test: 0.85452 | [acc] train: 0.6972 / test: 0.7049\n",
      "[  2] train: 0.03098 / test: 0.64783 | [acc] train: 0.7961 / test: 0.8078\n",
      "[  3] train: 0.02159 / test: 0.60947 | [acc] train: 0.8249 / test: 0.8317\n",
      "[  4] train: 0.01679 / test: 0.61779 | [acc] train: 0.8123 / test: 0.8190\n",
      "[  5] train: 0.01792 / test: 0.49727 | [acc] train: 0.8710 / test: 0.8768\n",
      "[  6] train: 0.01507 / test: 0.46659 | [acc] train: 0.8755 / test: 0.8773\n",
      "[  7] train: 0.01356 / test: 0.34264 | [acc] train: 0.9203 / test: 0.9171\n",
      "[  8] train: 0.01514 / test: 0.43176 | [acc] train: 0.8641 / test: 0.8618\n",
      "[  9] train: 0.01195 / test: 0.34641 | [acc] train: 0.8895 / test: 0.8901\n",
      "[ 10] train: 0.01041 / test: 0.27605 | [acc] train: 0.9142 / test: 0.9125\n",
      "[ 11] train: 0.00852 / test: 0.18095 | [acc] train: 0.9522 / test: 0.9477\n",
      "[ 12] train: 0.01152 / test: 0.20525 | [acc] train: 0.9386 / test: 0.9413\n",
      "[ 13] train: 0.01021 / test: 0.18752 | [acc] train: 0.9442 / test: 0.9468\n",
      "[ 14] train: 0.00956 / test: 0.15989 | [acc] train: 0.9510 / test: 0.9525\n",
      "[ 15] train: 0.00903 / test: 0.14371 | [acc] train: 0.9558 / test: 0.9567\n",
      "[ 16] train: 0.00781 / test: 0.16895 | [acc] train: 0.9483 / test: 0.9483\n",
      "[ 17] train: 0.00693 / test: 0.14990 | [acc] train: 0.9554 / test: 0.9558\n",
      "[ 18] train: 0.00834 / test: 0.17284 | [acc] train: 0.9491 / test: 0.9499\n",
      "[ 19] train: 0.00771 / test: 0.15361 | [acc] train: 0.9566 / test: 0.9566\n",
      "[ 20] train: 0.00724 / test: 0.13984 | [acc] train: 0.9575 / test: 0.9589\n"
     ]
    }
   ],
   "source": [
    "history_bn = train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plotting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "history = np.array(history)\n",
    "history_bn = np.array(history_bn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f13652634d0>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhIAAAFkCAYAAAB1rtL+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3Xlc1VX+x/HXAQF3TcV933FFMStTczdLzTHTNFMzp2zP\nZmraJ2vGlqmxmalfVpppLq2aaZqJmmaFJop7qbnigoKCCwIC5/fHFxBRUNYvF97Px+M+uPd7zz3f\nz4Xyvu/3e77nGGstIiIiIjnh5XYBIiIi4rkUJERERCTHFCREREQkxxQkREREJMcUJERERCTHFCRE\nREQkxxQkREREJMcUJERERCTHFCREREQkxxQkREREJMeyHSSMMV2MMd8YYw4ZY5KNMQOv4jU3GWPW\nG2POGWN2G2Puv0yb240x24wxccaYrcaYQdmtTURERApWTo5IlAHCgAeBKy7UYYypD3wLrAICgVeB\n/xpj/pSuzQ3Ap8DHQBtgFvC5MebaHNQnIiIiBcTkZtEuY0wyMMha+00WbV4H+ltrW6bb9h7Qxlp7\nY8rjT4Fy1tpb07VZApyw1t6V4wJFREQkXxXEGInrge8zbFsKdDDGeKc8viGTNp3yuTYRERHJhRIF\nsI/qQESGbREp+66Scj+zNtUz69QYUxnoC+wD4vKoVhERkeKgJFAfWGqtjcpNRwURJPJLX2C220WI\niIh4sLuAObnpoCCCxFGgWoZt1YBEIPIKbY5m0e8+gFmzZhEQEJD7KsV1EyZMYPLkyW6XIXlEf8+i\nRX/PomXHjh2MHDkSUj5Lc6MggsQvQP8M2/oC6621Sena9Ab+k65NH+DnLPqNAwgICKB9+/Z5VKq4\nqUKFCvpbFiH6exYt+nsWWbkeGpDtIGGMKQM0BkzKpobGmLY4V1gcNMa8CtS01o5OeX4K8JAx5i3g\nQ5wBlPcAd6br9j/AKmPMU8ACYBDQE7gxB+9JRERECkhOrtroAGwEQnHmkXgL2ABMTHm+OlAntbG1\ndh9wC3BTyuueAx6x1n6drs0vOMFiDLAJGAUMtdauz0F9IiIiUkCyfUTCWruKLAKItfaey2z7ESeA\nZNXvPGBedusRERER92itDSk0hg8f7nYJkof09yxa9PeUzChISKGhf6iKFv09ixb9PSUzChIiIiKS\nYwoSIiIikmMKEiIiIpJjChIiIiKSYwoSIiIikmMKEiIiIpJjChIiIiKSYwoSIiIikmMKEiIiIpJj\nChIiIiKSYwoSIiIikmMKEiIiIpJjChIiIiKSYwoSIiIikmMKEiIiIpJjChIiIiKSYwoSIiIikmMK\nEiIiIpJjChIiIiKSYwoSIiIikmMKEiIiIpJjJdwuQERERODsWdi6FbZsgc2bYd8+WLAAjHG7sqwp\nSIiIiBSgpCT4448LgSH15549YC14eUGTJtCmDcTFQalSblecNQUJERGRfBIZeXFY2LLFOepw7pzz\nvL+/ExgGDoTWrZ37LVoU/vCQnoKEiIgUGUlJcPAg7NoFO3fC7t3OtjJloGxZ52f6+5ltK1kye6cU\n4uJgx45LjzIcPeo87+cHLVs6QeHOO52frVtDtWr583soSAoSIiLiUax1PqBTw0L6n7t3Q3y8087H\nBxo0cD7Ez5xxxiCcOQOxsVfehzFXFz6io53AsHOnE1jA2Wfr1jBu3IXA0LgxlCiin7hF9G2JiIin\nO3Hi0qCQ+vPMGaeNMVC/vjOmoHt3uO8+aNrUeVyv3uU/vJOTnVMLqeEiNWCk/3mlbVFRzv0yZaBH\nD3jsMSc0tGoF5coV6K/JdQoSIiKSJ6x1vpUnJUFiYtY/09+Pi4O9ey8NC1FRF/quWdMJBx06wPDh\nF8JCw4bOaYjs8PK6cERBck9BQkSkmEtOdgYFHj7s3A4dunA/9RYZeeVwkJycuzoqV3bCQdOmcOut\nF8JC48bF71u+J1GQEBEpoqyFmJjMw0HqtiNHnDCQyhhnEGDNms6tQwfn6gIfH+dUgbe3c0u9n9tt\nPj7OaYjKld37XUnOKUiIiHi4qChYvBjCwi4NDamXGaaqVOlCQGjRAnr1uvC4Zk2oVcsJEUV1YKDk\nPf2nIiLiYayF33+HhQvhm2/g55+d0wpNmkDt2lC3Llx//cXhoGZNqFEj++MJRK5EQUJExAOcPw8/\n/eQEh4ULncscS5WC3r3h/fedMQU1arhdpRRHChIiIoVUdDR8950THpYscR7XqAEDBsDkydCzp2fN\ngChFk4KEiEgh8scfF05Z/PijMwgyMBAefdQJEO3bO5cviudLtskcP3ucw6cPc+j0IQ6fPnzRLepc\nFD+P/RlTyFftUpAQEXFRUhKEhFwIDzt2gK+vM8nRf/8L/ftDnTpuVynZYa0lOi4604CQuu3omaMk\nJl+4XMbLeFGtTDVqlqtJzXI1CawWyPnk8/h6+7r4bq5MQUJEpICdPg3ff++Eh2+/deZo8Pd3xjn8\n85/OuIeyZa+yr/jTbI7YTNjRMEr7lGZwwGAqlKyQv29AiE+MZ/X+1WyK2HRJQDh8+jBxiXEXta9c\nqjK1yteiZrmatPJvRZ+GfdICQ+r2qmWqUsLL8z6WPa9iESlQ1jprF5w7l/UtNvbix9Y6Vwhc7c3P\n78J9X9/sLZhUEBITnd9DfDwkJFy4f7lbZs/HxsKaNbBypdOmZUtnPYYBA+C665x5FTJjrSX8VDhh\nR8PYFLGJsKNhhB0N44+TfwDg4+VDYnIiDy1+iD8F/IkxbcfQo0EPvL2y6NRlicmJ/LDvB1bsXUFL\n/5b0aNCDGuUK74jRiDMRLN61mEW7FvH9H99zJuEMZX3LUrt8bWqWq0n9ivXpVLvTReGgZrmaVC9b\nnZIliu7lMgoSIsXI2bPOAkNhYbBpk/NNOGMAuNzN2qvfh5+fMwDQmAsBJDuvT5VV6PD1dfq01rns\nMfV+Zrcrtcn4/OXCQG5mbfTxcX4vfn7Qti288YYTHho2vHz780nn2RG5wwkNRzcRFuGEhhPnTgBw\nTclrCKweyMBmA2lbrS2B1QMJ8A/g2NljzNo8i4/DPmbOljnUKleLUW1HMbrtaJpVaZbzN5CHUsPD\n59s+Z/5v84mMjaRK6SpExkYC0MK/BT0b9KRng550q9/N1aMr1lo2RWxi0c5FLNy5kHWH1mEwXF/7\nep7p/Az9m/anddXWhX4MQ34zNgf/hxtjHgT+CtQAtgITrLVrsmj/EPAQUB/YD0yy1n6S7vnRwHTA\nAql/EQuUstYmZNJneyA0NDSU9u3bZ/s9iBR1UVGwcePFt507nQ/EEiWcb8M1ajgf+pndSpfO+vmM\n7UqWvHQgoLXOt/m4uLy7xcc7QcXLy/mZ8Zbb7b6+Fz74098y257Vcz4+WQ+OjI6LdsJCuiMN245v\nIyHJ+aev4TUNCaweSGC1QNpWd0JDnfJ1svzwstay7tA6Pg77mE+3fUp0XDTX176eMW3HMKzVMCqW\nrJgX/4ldtcuFhwYVGzC05VDuaHEH7Wu053jscVbsXcHyPctZvnc5e6P34mW8uLbmtU6waNiTTnU6\n5fs3+3Pnz7Fi7woW7lzIt7u+JfxUOOV8y9G3cV/6N+lPvyb9qFqmar7WUBA2bNhAUFAQQJC1dkNu\n+sp2kDDGDANmAuOBn1N+jgMCrLXhl2n/APBqSpv1wHXAh8Bwa+23KW1GA28DTbkQJLDWHsuiDgUJ\nEZwP6gMHLoSFsDDn58GDzvNlyjij/tu1u3Br0cL5kJOCdTDmIKFHQtNOS2yK2MS+6H0A+Hn70apq\nKyc0pNzaVGtDeb/yudpnXGIc3/z+DTM2zeC73d/h4+XDoOaDGBM4ht4Ne+fbqY+rCQ9ZhaE9J/ek\nhYoVe1dwPPY4JUuU5MY6N6YFi6AaQXlS/6FTh1i0cxGLdi1i+Z7lnEs8R8NrGjKg6QAGNB1Al3pd\nCv2Ax+xyO0iEAOuttQ+n27YdmG+tfe4y7X8C1lhr/5Zu22Sc4rumPB4NTLbWVspGHQoSUuwkJTkz\nGqY/yhAW5iy3DM6AvfSBoV07Z8EjXS7ojsOnD7Ny70pW7nNue07uAaBK6SppRxlSQ0OzKs3yfaDd\nkdNHmLV5FjM2zWDb8W3UKFuDu9vczejA0bTwb5Hr/lPDwxfbvmDeb/OyHR4yk2yT2XpsK8v3LCd4\nbzCr96/mTMIZKvhVoHuD7mmnQppXaX5V/SfbZNYfXu+Eh52L2Hh0I97Gm851O9O/aX/6N+1Ps8rN\nivQpC9eChDHGB4gFhlhrF6Tb/jbQ1lrb/TKvWQ98a639e7ptrwITgDLW2qSUIPEhcBjwBsKAF6y1\nYVnUoiAhRVpyshMU1q+/cJRh8+YLayfUr39paKhZs/ANUixOjp45yg/7fkgLD7tO7AKgpX9Lutfv\nTvcG3bm+9vXUKFvD1Q8pay2hR0KZETaDOVvncOLcCa6teS1jAsdwZ6s7qVTqqr/T5Vt4yMr5pPOs\nO7SO5XudIxa/HPyF88nnqVG2Bj0b9kwLFnUqXLhu9kzCGZb9sYxFOxfx7a5viTgbwTUlr6Ffk370\nb9Kfvo37Zut9ezo3g0QN4BDQyVobkm77M8Aoa23AZV7zT2AMMMBau8EY0wFYBPgDNa21EcaY64BG\nwBagPPA4cAvQxlr7Rya1KEhIkZOU5Kyb8OWX8NVXzuJL3t4QEHBxYAgMhIoFe5pbLuP42eP8sO8H\nJzzsW8mOyB0ANK/S3AkO9btzU/2bCvU59fjEeBbtXMSMTTNYvGsx3l7eDGw2kNFtR3Nz45sve5Qk\ns/BwR4s7GNpyaL6Eh6ycTTjLjwd+TDsVEnY0DIulSaUmdKvfjf0x+/lh3w8kJCUQUCUg7ahDpzqd\nPPJyy7zgaUGiJPAOcDfgBRwFZgFPAdWttccv8xoDbABWWWsfz6SW9kBo165dqVDh4lG9w4cPZ/jw\n4Vf9vkTclJTkzGCYGh6OHnUWWbr9dhg8GDp21DTIhcWJcydYtW9V2qmKrce2AqR9YHWv351u9bsV\n6ksYsxJxJoI5W+bw8aaP2RyxmWplqjGyzUhGtx1NgH9AoQoPWYmKjWLlvpUs37OcVftXUbNcTQY0\nHUD/pv1pVKmR2+UVuLlz5zJ37tyLtsXExLB69WrwhFMb6dp4A9WAI8D9wGvW2ky/UxljPgBqWWtv\nzeR5HZEQj5WYCKtWwRdfwPz5cOyYM3vhkCFwxx3OnAKePK7hdPxpTiecpma5mm6XkivRcdGs3r86\n7VTF5ojNWCwNKjZIO1XRrX43apev7XapeS7saBgfh33M7C2ziYyNpIxPGc6eP1tow4NkT14ekcjW\nMR1r7XljTCjQG1iQ7qnewNdXeG0SzhgIjDF3AguvsLtAYHN26hMpzM6fhxUrnCMP8+c7l2fWrw+j\nRzsB4tpri8b4hqTkJG6efTM/H/yZVlVbcXOjm+nXpB+d63Yu9CPfY+Ji+OngT2nBYePRjSTbZOpW\nqEv3+t2ZcP0EutXvRr2K9dwuNd8FVg/k7Zvf5o3eb7Bk1xI2R2zmlia3KDzIJXJycujfwMyUQPEL\nztGFOsB7kDaQsqa1dnTK4yZAR2AtUAl4AmgJjErt0BjzIhAC7MIZI/EY0BZ4IEfvSqSQSEiA4GDn\nyMOCBXDyJDRqBH/+sxMe2rcvGuEhvffWv8fPB3/mjV5vsD1yO7O2zOLNX96krG9ZejboSb/G/ejX\npB91K9R1u1SiYqP48cCPrN6/mlX7VxF2NIxkm0ytcrXo3qA7D3R4gO4NutOgYoNi++Hp6+3Lbc1v\n47bmt7ldihRS2Q4S1trPjTGVgBe4MCFVv3RzSFTHCRapvIG/4MwRcR5YiTPG4kC6NhWB91NeGwNs\nBLpYa0OzW5+I2+LinHUUvvzSWYQpJgaaNYOHHnLCQ5s2RS88pDoYc5Bnlj/D/UH38+SNTwLOpXab\njm7iu93fsWT3Eh5a/BBJNomAKgFpoaJL3S74lcj/iS0izkSkhYbV+1ez5dgWAOpWqMtN9W7iwQ4P\n0rVeVxpXalxsg4NIduVoZsvCQGMkpDA5dw6++84JDwsXOosytWzpBIchQ5z7Rf1zyVrLwE8HEno4\nlB0P7ch0auPouGiC9wSzZNcSvvvjOw6fPkxpn9L0aNDDCRaN+9HgmgZ5UlP4qXBW7VuVFh5+j/od\ngMaVGnNTvZvoWq8rN9W7qVicqhBJz7UxEiKe6ORJZx6GxETnConUn+nvZ7XtSu0PHIDFi511LNq0\ngSefdMJDwCXXMBVtX2z/gkU7FzFv6Lws10eoWLIiQ1oMYUiLIVhr2XJsC0t2LWHJ7iU89t1jPJT8\nEM0qN6Nf437c3Phmbqp/01VNi2ytZW/03rTQsGrfKvZG7wWc9Ru61+/OS91eomu9rh4/CFSkMNER\nCSnSwsKgf39nPoar5eXlzN1QooTzM/39jD+9veGaa2DgQOdyzaZN8++9FGYnzp0g4N0AOtftzFdD\nv8pxP6fiTxG8JzjtNEj4qXBKlShF9wbd045WpF6+Z61lZ9ROJzSknKoIPxWOwdCmWpu0Iw5d63XF\nv4x/Xr1VkSJBRyRErsK338KwYc74hMWLoXz5yweBjNuK+imI/PDX7/9KfGI8/+v3v1z1U96vPIMD\nBjM4YDDWWrYd35Z2tOKJpU/wyJJHaFypMS38W7A2fC0RZyPwNt60r9GeYS2HcVO9m+hctzPXlLom\nj96ZiFyJgoQUSe+8A4895izVPHu2s3CV5I/le5YzPWw67/d/P09PGRhjaFW1Fa2qtuLJG5/kdPxp\nVuxdwZLdS9h1Yhdj242la72u3FjnRsr5lcuz/YpI9ihISJGSlARPPAH//S9MmAD/+pdzlEHyR+z5\nWO5fdD9d63VlXPtx+bqvcn7ldBmiSCGkICFFxpkzMGKEc0rj3XfhwQfdrqjom/jDRMJPhfPtiG/x\nMh48FaeI5JiChBQJhw45pzF27YJFi6BfP7crKvo2HtnIW7+8xcRuE2lWpZnb5YiISxQkxONt2gS3\n3uoMklyzBtq2dbuioi8xOZFxC8fRwr9F2sRTIlI86VikeLTFi6FzZ6hWDdauVYgoKP8J+Q8bj2xk\n6sCphX79DBHJXwoS4rHefdc5ndGjB6xeDTU1x1CB2HNyDy+sfIFHr3uUjrU6ul2OiLhMQUI8TlIS\nPP44PPwwPPoozJunyzsLirWW+xfdj38Zf/7R4x9ulyMihYDGSIhHSX9lxjvvOAthScH5ZPMnBO8J\nZvGIxZT1Let2OSJSCChIiMc4fNg5lbFzp7Mw1i23uF1R8XLs7DEmLJ3AiNYj6NdEl8WIiENBQjzC\npk3OmhnW6soMtzz+3eMYDG/3fdvtUkSkENEYCbmi+HgID4dt2yAhoeD3v2SJc2WGv7+uzHDL4l2L\nmbt1LpP7TtYCWCJyER2RKIbOn4fISDh+HI4du/Azs/unTl14benScOON0L27cwsKAh+f/Kv1//4P\nHnnEmSdizhwoq9PyBe5Mwhke+PYBejfszcg2I90uR0QKGQWJIiQpCX7+GY4cyTocnDhx6WtLlYKq\nVZ1v/VWrOsthd+588bbSpWHdOli5EiZNgmefdT7Yu3SBbt2cYNG+fd6sbZGUBE8+CZMnO4tvvfWW\n1sxwy/MrnicyNpIfRv+A0dKoIpKBgkQR8sAD8OGHzn1f3wsBoGpVqFcPrr324m2p9/39ncsnr+Yz\n4qabnA/48+chNBR++MEJFhMnwt/+5izV3bXrhSMWbduCVzZPoJ0961yZsWgR/O9/zmWe4o614Wv5\n79r/8q/e/6LBNQ3cLkdECiFjrXW7hhwxxrQHQkNDQ2nfvr3b5bju44/hnnucSZruusv5QC/IL48J\nCfDrr06oWLnSOTISFwfXXHNxsGjVKutgkf7KjE8/dU5piDsSkhII+iAIP28/QsaFUMJL3ztEiooN\nGzYQFBQEEGSt3ZCbvvQvQxGwaZNzNOLee91b8dLX1xk7ceON8PzzzgDNtWsvBIunnnLCRpUqzlGN\n1GAREHAh8Gze7AQHa+HHHyEw0J33Io5//fQvdhzfwfr71itEiEim9K+Dh4uOhttvh+bNndMAhYWf\nn3MkomtX+Pvf4dw5CAm5ECwmTHBOj1St6oyvaN0aXn8dmjRx5oioVcvtd1C8/Rb5Gy+vfpm/dvor\ngdWV6EQkcwoSHsxa53RGZCR8/70zYLKwKlXqwlEIcMZB/PyzEyp++MGZ5vqWW2D2bF2Z4bZkm8x9\nC++jTvk6/P2mv7tdjogUcgoSHuzNN+Hrr2HBAmjY0O1qsqdMGejd27mBc3QiPy8jlas3dcNUfjzw\nIytGraCUTyFOpyJSKGhCKg+1ejU884xzpcTAgW5Xk3sKEYXD4dOHeXLZk4wNHEv3Bt3dLkdEPICC\nhAc6cgSGDXPmb/iHFmCUPPTIkkcoVaIU/+rzL7dLEREPoVMbHiYxEe6807nS4dNPoYT+gpJH5u2Y\nx7wd8/h8yOdUKlXJ7XJExEPoY8jDPPss/PSTM0ixWjW3q5GiIjoumocXP8yApgMY0mKI2+WIiAfR\nqQ0P8vXX8K9/OZdJdunidjWeY8XeFXz929dul1GoPR38NGcSzvB/t/6fpsEWkWzREQkPsXs3jB4N\ngwfDE0+4XY3niImL4c4v7yQxOZG+jfrqKoTLWL1/Ne+Hvs+7t7xL7fK13S5HRDyMjkh4gNhYZ9Kp\natVg+vSCnfra0036cRKnE05zMu4kn2/73O1yCp24xDj+vPDPdKrTifEdxrtdjoh4IAWJQs5aeOgh\n2LULvvrKWUNDrs6ek3t4e+3b/O3Gv9GnUR+mhE5xu6RC55+r/8nek3v5cMCHeBn9cyAi2ad/OQq5\nadOcBbmmTHGmkZar93Tw01QpXYUnOz3J+KDxhISHEHY0zO2yCo0tEVt47afXeK7Lc7Twb+F2OSLi\noRQkCrENG5wltO+/H0aNcrsaz7LmwBq+2P4Fk3pMooxvGfo37U/NcjV5f/37bpdWKJyKP8WYBWNo\nUqkJT3d+2u1yRMSDKUgUUidPwpAhzrLbb7/tdjWeJdkmM2HpBIJqBHF327sB8PH2YVy7cczaMovT\n8addrtBdp+JPcfOsm/njxB/MGjwLvxJ+bpckIh5MQaIQSk52jkBER8OXX0LJkm5X5FnmbJnD+sPr\nmdx38kXn/f8c9Gdiz8cyZ8scF6tzV2qI2H58O8vuXkb7Gu3dLklEPJyCRCH0+uuwaBF88gnUr+92\nNZ4l9nwszyx/htsDbqdLvYsn26hdvjYDmg7gvfXvYa11qUL3xMTF0HdW37QQcW2ta90uSUSKAAWJ\nQmblSnj+eed2661uV+N53vz5TY6dPcbrvV6/7PPjO4xnU8Qm1h5aW8CVuSs1RPwW+RvBo4IVIkQk\nzyhIFCKHDjnraPToAS+95HY1nufQqUO8/tPrPNrxURpVanTZNn0a9aF+xfpMWV98LgWNjoumz6w+\n7IzaSfDdwXSo2cHtkkSkCFGQKCTOn3dW9PTxgTlzwNvb7Yo8z/Mrn6e0T2me6/pcpm28jBf3B93P\nZ9s+48S5EwVYnTui46Lp80kfdkXtInhUMEE1g9wuSUSKGAWJQuJvf4O1a+GLL8Df3+1qPM+GIxuY\nETaDid0mUrFkxSzbjm03lqTkJGZumllA1bkjNUTsPrGb4FHBGlgpIvkiR0HCGPOgMWaPMeacMeZX\nY0znK7R/yBiz3RgTa4zZYYy5+zJtbjfGbDPGxBljthpjBuWkNk/05ZcweTK89RbccIPb1Xgeay0T\nlk4gwD+A+4Luu2L7qmWqcnuL25myfkqRHXR58txJen/Smz9O/sHyUcsVIkQk32Q7SBhjhgGTgVeA\nQGANsMQYc9nVfowxDwD/BF4EWgAvAe8aY25N1+YG4FPgY6ANMAv43BhT5EeE/f47jB0LQ4fCI4+4\nXY1n+vq3r1m9fzVv9XmLEl5Xtw7d+KDx/B71O6v2r8rn6gpeaojYc3IPy0ctp12Ndm6XJCJFmMnu\nNzJjTAiw3lr7cLpt24H51tpLTk4bY34C1lhr/5Zu22QgyFrbNeXxp0A5a236cLEEOGGtvSuTOtoD\noaGhobRv75nfts6eheuug6QkWLcOypVzuyLPE58YT8v/a0mTyk1YcteSq36dtZYW/9eCNtXa8NmQ\nz/KxwoKVGiL2Ru9l+ajlBFYPdLskESmENmzYQFBQEDifxRty01e2jkgYY3yAIGBZhqe+Bzpl8jI/\nIC7DtjigozEmdUjhDSl9pLc0iz49nrUwfjzs3euc2lCIyJl31r3Dvuh9vNn7zWy9zhjD+KDxzNsx\nj4gzEflUXcE6ce4EvT7pxb7ofawYtUIhQkQKRHZPbVQBvIGM//JGANUzec1SYFzKEQSMMR2AewCf\nlP5IeW12+vR4778Ps2bBhx9Cy5ZuV+OZImMjeWX1K9wXdB8tq2b/lziq7ShKeJXgo40f5UN1BevE\nuRP0mtmLAzEHWDF6BW2rt3W7JBEpJgriqo1XgCXAL8aY88B8YHrKc8kFsP9CZ/16eOwxZ3nwESPc\nrsZzvfTDS1gsE7tNzNHrryl1DXe2upMPNnxAUnJSHldXcKJio+g5sycHTx1kxagVtKnWxu2SRKQY\nubqRaRdEAklAtQzbqwFHL/cCa20czhGJ+1PaHQHuB05ba4+nNDuanT7TmzBhAhUqVLho2/Dhwxk+\nfPiVXuqKqChnMa7AQOcqDcmZ7ce3M2X9FF7t+Sr+ZXJ+vewDHR7g47CP+f6P7+nXpF8eVlgwomKj\n6PVJL8JPhbNi1ApaV9Na8yJysblz5zJ37tyLtsXExORZ/3k12HIb8PXlBltm0scPwEFr7d0pjz8F\nylpr+6drsxg4WZQGW0ZEwG23we7dzhLhdeu6XZHnumX2Lfwe9TvbH9yeq9UrrbUEfRBEnQp1WHDn\ngjysMP9FxkbSa2YvDp8+zIrRK2hVtZXbJYmIh8jLwZbZPSIB8G9gpjEmFPgF5+hCHeA9AGPMq0BN\na+3olMdNgI7AWqAS8ATQEhiVrs//AKuMMU8BC4BBQE/gxhzUVyht3Qr9+0N8PCxZohCRG0t3L2XJ\n7iV8eceXuV4C2xjD+A7jeeDbBzgQc4C6FTzjDxMZG0nPmT05cvqIQoSIuCrbYySstZ8DjwMvABuB\nzkA/a21CsM4IAAAgAElEQVR4SpPqOMEilTfwFyAMZ+ClL9DJWnsgXZ+/AHcCY4BNOCFjqLV2fXbr\nK4yWLIFOnaBiRecyz2uL/OwY+ScxOZG/fP8XutTtwuCAwXnS5/BWwynjU4apG6bmSX/5LTVEHD1z\nlJWjVypEiIircnJEAmvtFOCyqx5Za+/J8Pg34IrnHqy184B5OamnsLIW/vc/mDDBWclzzhwoW9bt\nqjzb1A1T2XZ8G7/++VeMMXnSZzm/coxsM5KpG6byQtcX8PH2yZN+88Pxs8fpObMnEWcjWDl6JS38\nW7hdkogUc1prI58kJsLDDztXZ0yYAPPnK0TkVkxcDC+ufJFRbUfl+QqW4zuM58iZIyzcuTBP+81L\nx84eo8fMHhw7e0whQkQKDQWJfBAd7RyB+OAD5/bmm1rNMy9M+nESZxLOMKnHpDzvu021NnSq06nQ\nLi9+7OwxeszowfGzxxUiRKRQUZDIY3v2OOMh1q2DpUvhz392u6KiYc/JPby99m2euvEpapWvlS/7\nGB80nmV7lrH7xO586T+nUkNE1LkofhjzAwH+AW6XJCKSRkEiD61Z46ydcf68syR4jx5uV1R0/C34\nb1QpXYUnOz2Zb/sY0mIIlUpV4oPQD/JtH9kVcSaC7jO6E3UuipWjV9K8SnO3SxIRuYiCRB755BPo\n2dOZ7jokBJo2dbuiomPNgTV8uf1LXu35KmV8y+Tbfkr5lGJM2zF8tPEj4hIzLg9T8E6cO0GPmT04\nee4kP4z+QSFCRAolBYlcSk6G55+HUaNg5Ej4/nuoXNntqoqOZJvMhKUTCKoRxMg2I/N9f/cF3UfU\nuSi+2v5Vvu8rK0nJSYz4akTaJZ7NqjRztR4RkcwoSORCbCwMGwaTJsEbb8DUqeDr63ZVRcvszbNZ\nf3g9k/tOxsvk/3+uzao0o0eDHkwJdXfQ5QsrX2DZnmV8evunChEiUqgpSOTQkSNw002weDHMmwdP\nPgl5NK2BpIg9H8szy5/h9oDb6VKvS4Ht94EOD7DmwBq2HttaYPtM76vtX/Hqmld5teer9G7U25Ua\nRESuloJEDoSFQceOTphYswYGDXK7oqLpzZ/f5HjscV7v9XqB7ve2ZrdRvWx13l//foHuF5zFyMYs\nGMMdLe7I14GlIiJ5RUEim775Bjp3hmrVnEs827Vzu6Ki6dCpQ7z+0+s82vFRGlVqVKD79vH24d52\n9zJz80zOJpwtsP3GxMUw6NNB1K9Yn49u+yjPZu4UEclPChJXyVpn2e9Bg6BvX1i9GmrWdLuqouu5\nFc9R2qc0z3d93pX9/7n9nzkdf5q5W+deuXEeSLbJjJw/kuOxx5k/bD5lfTUNqoh4BgWJq3D+PNx/\nP/z1r/D00/DFF1C6tNtVFV2hh0OZsWkGL3d7mQolK7hSQ72K9bilyS0FNtPlK6te4dud3zJ78Gwa\nV2pcIPsUEckLChJXcPIk3HwzfPyxc5s0Cbz0W8s31lqe+P4JWvi34M9B7k4LOr7DeEKPhLL+cP4u\nQrvw94W8tOolXu7+Mrc0uSVf9yUiktf0kZiFXbvg+uudwZXBwTB6tNsVFX3zf5vP6v2reavPW5Tw\nytHitHmmX+N+1K1QN1+PSuyM2snI+SO5rdltPNvl2Xzbj4hIflGQyMQPPzjTXXt5OYMqu3Z1u6Ki\nLz4xnqeWPcXNjW/m5sY3u10O3l7e3Nf+PuZunUt0XHSe9386/jSDPh1EjbI1mPmnmQUyT4aISF7T\nv1yX8fHH0Ls3BAXBL79Ao4K9aKDYemfdO+yL3sdbfd5yu5Q0Y9uNJSEpgVmbZ+Vpv9ZaxiwYQ/ip\ncL6+82vK+5XP0/5FRAqKgkQGS5bA2LEwZowz2VTFim5XVDwcP3ucV1a/wn1B9xWqJbJrlKvBoOaD\nmLJ+CtbaPOv3tTWvMW/HPGb+aabW0BARj+buSehCZvduGDECbrkF3n9fgyqzkpScxPnk8yQkJZCQ\nlMD5pHT3r2J7xueC9wYDMLHbRJff2aXGB42n1ye9WHNgTZ7MsPnd7u94bsVzPN/leQY112xmIuLZ\nFCRSnD7tzBHh7w+zZilEZHT49GGum3odkbGRJCQlkGyT86TfEl4l8PX2xc/bj3/3/Tf+ZfzzpN+8\n1L1Bd5pUasKU0Cm5DhJ7Tu5hxFcj6NekHy91eylvChQRcZGCBM5kU/fcA/v3w9q1Op1xOdM3TufE\nuRO80esN/Er44ePlg6+3L77evvh4p7ufje0+Xj4eMXujl/FifIfxPLP8Gd7u+3aOw87ZhLP86bM/\nUbl0ZWYPno23l3ceVyoiUvAUJIBXX4WvvoL586FF4Tk9X2gk22SmbZzGsJbDeOS6R9wuxxWj247m\n2eXP8nHYxzx5Y/bXwLDWMm7hOP448Qch40KoWFJpVUSKhmJ/AH/xYnj+eXjhBS2+lZmVe1eyN3ov\n97a71+1SXFO5dGWGthzK+6Hv5+i0zr9/+Tefbv2U6bdNp1XVVvlQoYiIO4p1kNi1yxlceeut8NJL\nbldTeE3bOI3mVZrTqU4nt0tx1fgO4/nj5B8s37M8W69bsXcFTwU/xVOdnuKOlnfkU3UiIu4otkEi\ndXBltWoaXJmVqNgovtrxFePajfOI8Qz56YbaN9C6amveW//eVb9mf/R+hn4xlJ4NejKp56R8rE5E\nxB3F8uPTWmeeiIMH4euvoYI760J5hNlbZpNsk7m77d1ul+I6YwzjO4znm9+/4dCpQ1dsf+78OQZ/\nPphyfuWYe/tcDa4UkSKpWAaJSZNg3jz45BMICHC7msLLWsvUDVO5rdltVC1T1e1yCoWRbUZSskRJ\npm2clmU7ay3jvx3PjuM7mD9sPpVLVy6gCkVEClaxCxLffusMrHzxRbjtNrerKdzWH17PlmNbGNd+\nnNulFBrl/cpzV+u7+HDDhyQmJ2ba7t1f32Xmppl8OOBDAqsHFmCFIiIFq1gFiV274K67oH9/+Pvf\n3a6m8Ju6YSp1ytehd8PebpdSqIzvMJ7wU+Es3rX4ss//uP9HJiydwOPXPc5dbe4q4OpERApWsQkS\nqYMrq1d3TmlocGXWziacZe7WudwTeI/O7WfQrkY7OtbqeNnlxcNPhTPkiyF0rtuZN3q/4UJ1IiIF\nq1h8nCYnw+jRGlyZHV9s/4IzCWe4p909bpdSKI0PGs93u79j78m9adviE+MZ8vkQfL19+WzIZ/h4\n+7hYoYhIwSgWQWLSJGfWylmzoLkWWrwqUzdMpVfDXtSvWN/tUgqlYa2GUaFkBT4I/SBt2yNLHiHs\naBjzhs7T4FQRKTaKfJD49ltnYOXf/w4DB7pdjWf4LfI3fjr4kwZZZqG0T2lGtx3NtI3TSEhK4IPQ\nD/hww4e8d+t7XFvrWrfLExEpMEU6SOzc6cxcOWCAEybk6kzbMI3KpSpzWzNd1pKV+4Pu53jscZ4O\nfpqHFz/Mgx0e1KkgESl2imyQOHXKGVxZo4YGV2ZHQlICMzbN4O42d+NXws/tcgq1AP8Abqp3E5ND\nJtOxVkcm3zzZ7ZJERApckVz9M3VwZXg4rFsH5cu7XZHnWLRzEcdjj3Nv++K7QFd2PNvlWWLPx/LF\nHV/g6+3rdjkiIgWuSAaJSZOcqzMWLNDgyuyaumEq19W6TitUXqU+jfrQp1Eft8sQEXFNkTvgnzq4\n8qWXNLgyuw7GHOS73d9pkKWIiFy1IhUkUgdXDhzoTIMt2fNx2MeU9inNsJbD3C5FREQ8RJEJEqmD\nK2vWhJkzNbgyu5JtMtM2TmNYy2GU8yvndjkiIuIhisQYidTBlYcOaXBlTi3fs5z9Mft1WkNERLKl\nSASJf/7TGVz5zTfQrJnb1XimaRun0cK/BdfXvt7tUkRExIPk6ASAMeZBY8weY8w5Y8yvxpjOV2g/\nyhizyRhz1hhz2BjzkTGmUrrnRxtjko0xSSk/U+9f8Xq61audWSsnTnQmnpLsi4yNZP5v87m33b0Y\nY9wuR0REPEi2g4QxZhgwGXgFCATWAEuMMbUzad8N+Aj4AGgBDAGuBT7M0DQGqJ7uVsNam3Clep5/\nHm67zfkpOTNr8yystdzd5m63SxEREQ+TkyMSE4APrbXTrbW/W2snAAeBBzJpHwTstda+a63db639\nGXgf6JChnbXWHrfWHku9XU0x/v4wY4YGV+aUtZZpG6cxqPkg/Mv4u12OiIh4mGx9/BpjfHCCwbIM\nT30PdMrkZcuAasaYfil9VAPuABZlaFfWGLPPGHPQGLPQGBN4NTW99ZYGV+bGukPr2HpsK/e200yW\nIiKSfdn9Hl8F8AYiMmyPwDkdcQlr7WZgFPCFMSYBOAKcAB5N1+w3YAwwALgTiAN+MsY0ulJB9etn\nq37JYNrGadStUJdeDXu5XYqIiHigfL9qwxhzPTADeBHnyEUN4E2c0xvjAKy1a4G16V7zM7ABeAR4\nPKv+J0yYQIUKFS7aNnz4cIYPH553b6KIOpNwhrlb5/LXG/6Kt5e32+WIiEg+mDt3LnPnzr1oW0xM\nTJ71b6y1V9/YObURCwyx1i5It/1toK21tvtlXvMp4GWtHZpu243AjzgDKjMe3Uht8wFQy1p7aybP\ntwdCQ0NDad++/VW/B7ngo40fMe6bcex7fB91K9R1uxwRESkgGzZsICgoCCDIWrshN31l69SGtfY8\nEAr0zvBUb+DnLPaRmGFbMmCBrK41DMQ5DSL5ZNrGafRp1EchQkREciwnpzb+Dcw0xoQCvwD3A3WA\n9wCMMa8CNa21o1Pafw1MN8aMB5YCNXEuH11rrT2a8poXgRBgF1AeeAxoS+ZXgkgubT++nZ8P/szn\nQz53uxQREfFg2Q4S1trPUyaTegFnvMNWoJ+1NjylSXWcYJHafo4xpjzwEM7YiGhgOfB0um4r4oyZ\nqI4zn8RGoIu1NjTb70iuyrQN06hSugoDm2mJVBERybkcDba01k4BpmTy3D3ZaZ/y/BPAEzmpRbIv\nISmBmZtnMqrNKPxK+LldjoiIeDBN41QMffP7N0TGRnJve80dISIiuaMgUQxN3TCVG2rfQAv/Fm6X\nIiIiHk5Bopg5EHOA7//4XsuFi4hInlCQKGamb5xOGd8yDG059MqNRURErkBBohhJSk7io7CPuLPl\nnZT1Let2OSIiUgQoSBQjy/cu50DMAZ3WEBGRPKMgUYxM3TCVlv4t6Viro9uliIhIEaEgUUwcP3uc\nr3/7mnHtx2FMVjOTi4iIXD0FiWJi1uZZGGMY2Wak26WIiEgRoiBRDFhrmbpxKoOaD6JK6SpulyMi\nIkWIgkQxEBIewvbj2xnXToMsRUQkbylIFAPTNk6jXoV69GzY0+1SRESkiFGQKGAHYg6w9djWAtvf\n6fjTfLr1U8a2G4uX0Z9bRETylj5ZCtDuE7u5bup1tH6vNaO/Hk34qfArvyiXPtv2GbHnY7kn8JJF\nWUVERHJNQaKAhJ8Kp9fMXlTwq8Dbfd9mya4lNP1fU1764SXOJpzNt/1O2ziNmxvfTJ0KdfJtHyIi\nUnwpSBSA42eP0/uT3lgsy+5exmPXP8auR3bx6HWP8uqaV2n6TlNmhM0g2Sbn6X63HttKSHgI97bT\ncuEiIpI/FCTyWXRcNH1n9eXkuZME3x2cdmSgQskKvNbrNX576Dc61+3MmAVjuPbDa1m1b1We7Xva\nhmn4l/ZnQLMBedaniIhIegoS+ehswln6z+nPvuh9LLt7GU0qN7mkTYNrGvDZkM9Yc88avI033WZ0\nY/Bng9l9Yneu9h2fGM8nmz9hVNtR+Hr75qovERGRzChI5JP4xHgGfz6YsKNhLLlrCa2rtc6y/Y11\nbyRkXAizB89m/eH1tHi3BX9Z+hei46JztP8Fvy8g6lyUTmuIiEi+UpDIB4nJiYyYN4JV+1axcPhC\nrqt93VW9zst4MaL1CH57+DdevOlF3g99n8b/bcw7697hfNL5bNUwdcNUbqxzIwH+ATl5CyIiIldF\nQSKPJdtkxn0zjm9+/4Yv7viC7g26Z7uP0j6leb7r8+x6ZBeDmg/i0SWP0mZKG77d+S3W2iu+fl/0\nPoL3BOtohIiI5DsFiTxkreXx7x5n5qaZzBw0M9eDHGuUq8HUgVPZcP8GapStQf+5/ek7qy9bIrZk\n+brpG6dT1rcsd7S8I1f7FxERuRIFiTz04soX+d+6/zGl/xSGtx6eZ/0GVg9k+ajlLLhzAfui9xH4\nfiD3L7yfiDMRl7RNSk7io7CPGN5qOGV9y+ZZDSIiIpejIJFH3vz5Tf7x4z94o9cb3Bd0X573b4xh\nYLOBbH1wK//u82++2P4FTf7XhNfWvEZcYlxau2V7lhF+Kpx72+u0hoiI5D8FiTzwQegHPLnsSZ7r\n8hxP3vhkvu7L19uXx65/jN2P7mZsu7G8sPIFmr/TnM+2fuYsF75hKq2rtubamtfmax0iIiKgIJFr\nc7fMZfyi8TzS8RFe6f5Kge23UqlKvH3z22x7cBttq7flzq/upNNHnVjw+wLGtR+HMabAahERkeJL\nQSIXFu1cxKivRzGq7SjevvltVz68m1ZuyoI7F7B81HLOnT9HyRIluav1XQVeh4iIFE8l3C7AU63c\nu5Ihnw9hYLOBTB041fUluns06EHofaHExMdQqVQlV2sREZHiQ0ckcmBt+FoGfjqQm+rfxJzBcyjh\nVTjymLeXt0KEiIgUKAWJbNoSsYV+s/vRtlpb5g2dh18JP7dLEhERcY2CRDbsPrGb3p/0pl7Feiwa\nsYgyvmXcLklERMRVChJX6WDMQXrN7EXFkhVZOnIpFUtWdLskERER1ylIXIVjZ4/R+5PeAASPCqZq\nmaouVyQiIlI4FI5RgoVYdFw0fWf1JSY+hh/v+ZHa5Wu7XZKIiEihoSCRhbMJZ7l1zq3sj97PqjGr\naFypsdsliYiIFCoKEpmIT4znT5/9ic0Rm1k+ajmtq7V2uyQREZFCR0HiMhKTExn+1XBW71/NdyO/\no2Otjm6XJCIiUigpSFzGw4sfZuHOhcwfNp9u9bu5XY6IiEihpas2MohLjOOjjR/xcreX6d+0v9vl\niIiIFGoKEhlsPLKR88nn6du4r9uliIiIFHoKEhmEhIdQqkQpWlfV4EoREZEryVGQMMY8aIzZY4w5\nZ4z51RjT+QrtRxljNhljzhpjDhtjPjLGVMrQ5nZjzDZjTJwxZqsxZlBOasutkEMhdKjZAR9vHzd2\nLyIi4lGyHSSMMcOAycArQCCwBlhijLnsTE3GmG7AR8AHQAtgCHAt8GG6NjcAnwIfA22AWcDnxphr\ns1tfboWEh3B97esLerciIiIeKSdHJCYAH1prp1trf7fWTgAOAg9k0j4I2Gutfddau99a+zPwPtAh\nXZvHgO+ttf+y1u601r4GLAcez0F9OXb49GEOxBxQkBAREblK2QoSxhgfnGCwLMNT3wOdMnnZMqCa\nMaZfSh/VgDuARena3JDSR3pLs+gzX6wNXwugICEiInKVsntEogrgDURk2B4BVL/cC6y1m4FRwBfG\nmATgCHACeDRds+rZ6TO/hISHUKd8HWqWq1mQuxUREfFY+T4hlTHmemAG8CLOUYcawJs4pzfG5bb/\nCRMmUKFChYu2DR8+nOHDh2e7r5BDGh8hIiJFy9y5c5k7d+5F22JiYvKs/+wGiUggCaiWYXs14Ggm\nr3kcWGqt/XfK463GmAeBH40xz1lrI1Jem50+00yePJn27dtfbf2ZSkxO5NdDv/KPHv/IdV8iIiKF\nxeW+XG/YsIGgoKA86T9bpzasteeBUKB3hqd6Az9nsY/EDNuSAQuYlMe/XKbPPln0mee2RGzhXOI5\nHZEQERHJhpyc2vg3MNMYE4oTAO4H6gDvARhjXgVqWmtHp7T/GphujBmPM4CyJs7lo2uttalHHP4D\nrDLGPAUsAAYBPYEbc/SuciAkPAQfLx/aVW9XULsUERHxeNkOEtbaz1Mmk3oBZ7zDVqCftTY8pUl1\nnGCR2n6OMaY88BDO2IhonEs7n07X5hdjzJ3AP4CXgT+Aodba9Tl6VzkQciiEttXbUsqnVEHtUkRE\nxOPlaLCltXYKMCWT5+7JTvt0beYB83JST14ICQ+hT8M+bu1eRETEI2mtDSAqNoqdUTs1PkJERCSb\nFCSAdYfWAZqISkREJLsUJHBOa1QpXYWG1zR0uxQRERGPoiDBhYmojDFXbiwiIiJpin2QSLbJrA1f\ny/W1dFpDREQku4p9kPg98ndi4mM0PkJERCQHin2QCAkPwWC4tta1bpciIiLicRQkwkNoWbUl5f3K\nu12KiIiIx1GQOBSi8REiIiI5VKyDxOn402w9tlXjI0RERHKoWAeJ9YfXk2yTFSRERERyqFgHibWH\n1lLerzwB/gFulyIiIuKRinWQCAkPoWOtjniZYv1rEBERybFi+wlqrSUkXAMtRUREcqPYBon9MfuJ\nOBuh8REiIiK5UGyDREh4CADX1b7O5UpEREQ8V7EOEo0rNaZK6SpulyIiIuKxinWQ0GkNERGR3CmW\nQSI+MZ6NRzdqoKWIiEguFcsgsfHoRhKSEnREQkREJJeKZZAICQ+hZImStKnWxu1SREREPFqxDRId\nanbAx9vH7VJEREQ8WrENEhofISIiknvFLkgcOX2E/TH7NT5CREQkDxS7ILH20FoABQkREZE8UOyC\nREh4CLXL16ZW+VpulyIiIuLximWQ0NEIERGRvFGsgkRiciK/Hv5VAy1FRETySLEKEluPbSX2fKyO\nSIiIiOSRYhUkQsJDKOFVgvY12rtdioiISJFQ7IJEYPVASvmUcrsUERGRIqHYBQmNjxAREck7xSZI\nnDh3gt+jftf4CBERkTxUbILEukPrAE1EJSIikpeKTZAICQ+hSukqNLymoduliIiIFBnFKkhcV+s6\njDFulyIiIlJklHC7gIKQbJNZe2gtf7nhL26XIiJyiQMHDhAZGel2GVKEVKlShbp16xbIvopFkNgZ\ntZPouGiNjxCRQufAgQMEBAQQGxvrdilShJQuXZodO3YUSJgoFkEiJDwEg+Hamte6XYqIyEUiIyOJ\njY1l1qxZBAQEuF2OFAE7duxg5MiRREZGKkjklZDwEFr4t6BCyQpulyIiclkBAQG0b69Zd8Xz5Giw\npTHmQWPMHmPMOWPMr8aYzlm0nW6MSTbGJKX8TL1tSddm9GXaJBljfHNSX0Za8VNERCR/ZDtIGGOG\nAZOBV4BAYA2wxBhTO5OXPApUB2qk/KwNnAA+z9AuJuX51FsNa21CduvL6EzCGbYc26IgISIikg9y\nckRiAvChtXa6tfZ3a+0E4CDwwOUaW2tPW2uPpd6AjkBF4ONLm9rjGdrm2vrD60m2yQoSIiIi+SBb\nQcIY4wMEAcsyPPU90OkquxkLBFtrD2bYXtYYs88Yc9AYs9AYE5id2jITEh5COd9yBFTRICYREZG8\nlt0jElUAbyAiw/YInNMRWTLG1AD6AR9meOo3YAwwALgTiAN+MsY0ymZ9lwgJD6FjrY54e3nntisR\nERHJoKBnthwDnAQWpN9orV1rrZ1jrd1irf0JGArsBB7Jzc6stRpoKSLiYbp3706PHj3SHp87d46J\nEyeyevXqS9q+9NJLeHl5ceLEiYIsEXAus5w4cSIHDhy4qvZu1pqfsnv5ZySQBFTLsL0acPQqXn8P\nMNNam5hVI2utNcb8CjS5UocTJkygQoWLL+scPnw4w4cPZ3/MfiLORihIiIh4kPfee++ix7GxsUyc\nOBFjDF27dr3oOWOMa0sfbN++nYkTJ9K9e/ermq/BrVrnzp3L3LlzL9oWExOTZ/1nK0hYa88bY0KB\n3lx8VKE38HVWrzXGdAMaAdOucneBwOYrNZo8eXKm116HhIcAcF2t665ylyIi4rbmzZtf9Nha61Il\nWbPWesT6TalfrtPbsGEDQUFBedJ/Tk5t/BsYZ4y5xxjT3BgzGagDvAdgjHnVGDPjMq+7F1hrrd2R\n8QljzIvGmD7GmAbGmLbGmI+Atql95lRIeAiNrmmEfxn/3HQjIiLZtGPHDry8vPjqq6/Stm3cuBEv\nLy9atWp1UduBAwfSoUOHtMfdunVLO7Wxf/9+qlatijEm7dSAl5cXY8eOvaiPo0ePMmLECCpWrEj1\n6tUZO3Ysp0+fvqhNfHw8zzzzDA0bNsTPz4/atWvz8MMPX/Lt3MvLi5dffvmS91S/fv20/c6YMYOh\nQ4em1evl5YW3tzczZ8684u/mwIED3H777VSoUIGKFSty9913X7LWSv369Rk4cCBLly4lKCiI0qVL\nExAQwPTp06/Yf0HL9syW1trPjTGVgBdw5obYCvSz1oanNKmOEyzSGGPKA3/CmVPicioC76e8NgbY\nCHSx1oZmt770ND5CRIqa2Fj47bf83Ufz5lC6dO76CAgIoEaNGgQHB3P77bcDsGzZMkqVKsWOHTs4\nevQo1atXJykpidWrV/PAAxdmEEj/Lb9GjRosXbqUvn37Mm7cOMaNGweAv/+FL4jWWoYMGcKwYcMY\nN24cW7Zs4emnn8bLy4upU6emtbvttttYuXIlzz77LJ07d2bz5s28+OKLhISE8Msvv+Dj45Ple0pf\nV//+/Zk0aRLPPfcc7733Hu3atQOgUaOsrxGw1jJ48GCGDh3KAw88wLZt23j++efZsWMHa9euxdvb\nO21fYWFh/PWvf+Xpp5+mWrVqTJ06lXvvvZcmTZrQuXOm80AWuBxNkW2tnQJMyeS5ey6z7RRQNov+\nngCeyEktmYlPjGfj0Y2MbDMyL7sVEXHVb79BHh2RzlRoKOTFbN09e/YkODg47XFwcDB33303X375\nJcHBwYwcOZK1a9dy6tQpevbsedk+fH19005f165dm44dO17SxhjDuHHjeOIJ52OkR48e7Nq1i+nT\np6cFiaVLl/L999/z5ptvprXr2bMntWvXZtiwYcycOZN77733qt9b5cqVadLEGcYXEBBw2boyc/vt\nt/Paa68B0KtXL6pWrcpdd93F559/ftEpiKioKH755Rdq1aoFQJcuXQgODmbOnDmeHyQ8wcajG0lI\nSl4XhIYAAA3TSURBVNARCREpUpo3dz7o83sfeaFnz57Mnj2bAwcOUK1aNdasWcODDz5IZGQky5Yt\nY+TIkQQHB1OyZEm6dOmSq30NGDDgosdt2rQhLi6O48eP4+/vz8qVKzHGMHr06Iva3XHHHYwdO5bl\ny5dnK0jklDGGESNGXLRt6NChjB49mpUrV14UJAIDA9NCBICfnx9NmzZl//79+V5ndhTZIBESHkLJ\nEiVpU62N26WIiOSZ0qXz5mhBQejVqxfWWpYtW0b9+vVJTEykR48eHD16lH/84x8ALF++nE6dOuHn\n55erfVWuXPmix6n9nTt3DnC+3ZcoUeKSdgDVq1cnKioqV/vPjurVL552ydvbm8qVK19Sw+Vq9fPz\nS3tPhUVBzyNRYELCQwiqEYSvd56s+yUiItlUq1YtmjZtSnBwMMuWLaNDhw6UL1+enj17cuTIEdat\nW0dISAi9evXK91oqV65MYmLiZQPD0aNHqVKlStpjPz8/4uPjL2mXV/M/HD168WwJSUlJREVFXTY4\neIIiHSR0WkNExF29evVixYoVLFu2jN69ewPQpEkTateuzYsvvkhiYuIVg0TGows50bNnT6y1zJo1\n66LtX375JWfPnr2ohvr167N588WzD6xYseKSq0D8/Pyw1marLmsts2fPvmjbZ599RmJiIt27d///\n9u4+tqq7juP4+0vLwK5KgQsUmQWmLGM6GavrxA15WOumM3SuYcoAUWd8QOMDGZuPbfEhLtGgxDkN\nJutYttWxrCu4aDpBYkQp27qM0dH6hMLKVrAdQR0sjvL1j3Opt11bbm9v77n38HklN3DPOffcb3P6\nu/3cc37n90t6P9kkkpc2Xvr3Sxw6cUhBQkQkZNdddx333HMPXV1dbNq0qXd5eXk5dXV1TJw4sc+t\nnwMpLCxk5syZbNu2jaVLlzJp0iRisRgzZ85Muo6Kigquv/567rzzTk6cOME111zDvn37qK2tpbS0\nlFWr/t8xf/Xq1VRXV1NTU8OiRYs4cOAAd999N0VFRX32efY21s2bN1NYWMj48eOZPXs2kyZNGrKW\nhoYG8vLyqKiooLW1lerqaubPn8/y5cuT/nmySSTPSOw9shdAQUJEJGRLly5lzJgxFBYWsmDBgt7l\n5eXlmFmfobAT9R/o6d5776WgoIDKykrKysrYsGHDsGtpbGxk3bp13Hfffdx4441s3LiRNWvWsHPn\nzj63fq5fv57169ezZcsWli1bRkNDA4888ghFRUV96po1axabNm1i3759LFmyhLKyMh5//PEhazAz\nGhoaaG9vp6qqitraWiorK2lqaiI/P7/PdoMNdpVtg2BZto4Ydi5mdiXQ0tLS8rqRLb+y4ys88NwD\ndKzrGPjFIiJZ4uwIgwN9lomkIpnfqYSRLUvd/ZmRvF8kz0iof4SIiEhmRC5InD5zmqdefEpBQkRE\nJAMiFyRaj7Vy8rWTChIiIiIZELkg0dzRTP6YfK6crmuNIiIioy2SQWLetHkUjB3hjDMiIiJyTpEL\nEnuP7NVlDRERkQyJVJA4fuo47V3tChIiIiIZEqkg8eSRJwENRCUiIpIpkQoSzR3NTH7DZN468a1h\nlyIiInJeiFaQOBIMRJVtw4eKiIhEVWSCxBk/w94OdbQUEckmtbW1jBkzJm1TcEv2iUyQ+Ev3Xzj+\n6nEFCRGRLDLU5FMSDZEJEs0dzRjGVW++KuxSREREzhuRChJzp8xlwvgJYZciIiL9HD58mKqqKiZM\nmEBRURGrV6+mq6urd/2sWbNYtmwZTU1NlJaWUlBQwNy5c6mrqwuxaklGdILEkWbePUOXNUREso27\nc/PNNzNnzhweffRRNmzYQGNjIzfccAM9PT1AcAnk2Wef5fbbb2fdunVs376defPmcdttt7F79+6Q\nfwIZSn7YBaTDK/99heeOPsfad60NuxQRkVF18rWTtHe1j+p7XBq7NO3TDFRVVXHXXXcBUF5eztSp\nU1m5ciVbt25lxYoVAHR3d7Nnzx5mzJgBwMKFC9mxYwcPPfQQ1157bVrrkfSJRJB4+sWnOeNn1NFS\nRCKvvaud0s2lo/oeLZ9qSevEh2bGrbfe2mfZLbfcwpo1a9i1a1dvkLjiiit6QwTAuHHjuOSSSzh0\n6FDaapH0i0SQaO5opvCCQi6bclnYpYiIjKpLY5fS8qmWUX+PdCsuLu7zPC8vj8mTJ9Pd3d27bPLk\nya973bhx4zh16lTa65H0iUaQONJM2Ywy8sbkhV2KiMioKhhbkNazBZnS2dnJ9OnTe5/39PTQ3d1N\nLBYLsSpJh5zvbOnuNHeoo6WISLZydx588ME+yx5++GFOnz7N4sWLwylK0ibnz0h0/qeTzv90qn+E\niEgWa2hoIC8vj4qKClpbW6murmb+/PksX7487NJkhHL+jMT+o/sBuPqiq0OuREREBmJmNDQ00N7e\nTlVVFbW1tVRWVtLU1ER+fn7vNoONgKmRMbNbzp+R2H9sPxdPvJipF04NuxQREemnpqaGmpoaALZt\n2zbodgcPHhxw+a5du0alLkmf3D8jcWy/LmuIiIiEJOeDRFtXmzpaioiIhCTng8TpntM6IyEiIhKS\nnA8SY/PGMq94XthliIiInJdyPkjMnTKXC/IuCLsMERGR81LOB4nLp1wedgkiIiLnrdwPEtMUJERE\nRMKS8+NIXD5VQUJEcl9bW1vYJUhEZPp3KeeDxLTCaWGXICKSslgsRkFBAatWrQq7FImQgoKCjE2I\nlvNBQkOnikguKykpoa2tja6urrBLkQiJxWKUlJRk5L1yPkhIdNTX17NixYqwy5A00fFMXklJScY+\n9FOl4ymDSamzpZmtNbODZnbKzJ4ys2uH2LbOzM6YWU/837OP/f22qzKz583sVTNrNbObUqlNcld9\nfX3YJUga6XhGi46nDGbYQcLMPgz8EPg2cAWwG/i1mV00yEu+ABQD0+P/XgS8DGxN2OcC4BfAfcA7\ngQeArWZ21XDrExERkcxJ5YzEl4Gfu3udu//J3b8MvAB8dqCN3f3f7n7s7AMoA4oIQsNZXwSecPfv\nu/uf3f0uYCfwpRTqExERkQwZVpAws7FAKfCbfqueAN6T5G4+Aexw9xcSli2I7yNR0zD2KSIiIiEY\nbmfLGJAHHO23/CjBZYshmdl04P3AR/qtKk5hn+NB915HyYkTJ3jmmWfCLkPSRMczWnQ8oyXhb+f4\nke4r03dtfAw4DmxLw75mAbr3OmJKS0vDLkHSSMczWnQ8I2kW8MeR7GC4QaIL6AH6jwI1DehM4vUf\nB+5399P9lnemsM8mYCXwD+DVJN5bREREAuMJQkTTSHdk7j68F5g1A0+7++cTlj0PNLr714d43WKC\nDpTvcPe2fut+ARS6+wcTlv0KOO7uK4dVoIiIiGRMKpc2NgL3m1kLsAf4NPAW4KcAZvY94M3uvqbf\n624D9vYPEXGbgN+Z2R0Elz1uAq4DrkmhPhEREcmQYQcJd99qZpOAbxKMDdEKvN/dO+KbFBMEi15m\n9ibgQwRjSgy0zz1m9hHgO8C3gL8Bt7j708OtT0RERDJn2Jc2RERERM5KaYhsEREREVCQEBERkRHI\nySAxnEnDJHuZWU2/idzOmNmLYdclyTOzhWa23cyOxI/fsgG2qY2vP2lmu8zssjBqlXM71/FMmIQx\n8TGiMQhk9JjZV83sSTP7l5kdNbPHzOySAbYbURvNuSCRwqRhkt1aCcYMKY4/Lg+3HBmmC4FngbXA\n6zpcmdmdBHPprAXeRTA2zG/M7MJMFilJG/J4xv2avm32A5kpTVKwEPgxcDVQTnCDxRNm9oazG6Sj\njeZcZ8tBxrE4ADw21DgWkn3MrAaodPcrw65FRs7MzgA3ufv2hGUvAhvd/Qfx5xcQDH9/h7v/PJxK\nJRmDHM86YIK73xxeZZIqM4sBx4D3uvvu+LIRt9GcOiORpknDJLvMiZ9SO2hm9WY2O+yCJD3ix7KY\nhPbq7v8Ffofaay5bHD9N/icz22xmU8IuSJJWRHCm6WVIXxvNqSDBCCcNk6zTDHwUeB/wSYJj+Ecz\nmxhqVZIuxQQfWmqv0fErgqkJlgDrgKuAnfEveZL9fgj83t0PxJ+npY1metIukV7unjjG+/Pxy1Z/\nA9YAPwqnKhEZjLs/kvD0QHyE438ANwKNoRQlSTGznwBvZxRGjM61MxIjnTRMspi7nwT2A3PCrkXS\nohMw1F4jy907gcOozWY1M/sx8EFgsbu/lLAqLW00p4KEu78GtAAV/VZVMMJpUCV8ZjYOmAu8dK5t\nJfu5+98JPox622u8I9ci4A9h1SXpE++89xbUZrOWmd1NMH/VEnc/nLguXW00Fy9tDDZp2M9CrUqG\nzcy+D/yS4BvNNOAbwBuBLWHWJcmL3yL2NoJvNQAXm9k84GV3f4HgEtXXzOyvwF+BrwGvAPVh1CtD\nG+p4xh+1wKMEwWE28F2CuwAey3ixck5mdg+wAlgGvGJmZ888nHD3V+P/H3EbzbnbPwHM7DPAHfx/\n0rAvubu+4eQYM6snuM85BvyToPPlN929PdTCJGlmtgjYxevHHNji7p+Ib1NNEPgnAnuBzyV09pIs\nMtTxJBhnoJFg/J4igjDxW6Da3Y9ksk5JTvwW3oH+yH/c3e9P2G5EbTQng4SIiIhkh5zqIyEiIiLZ\nRUFCREREUqYgISIiIilTkBAREZGUKUiIiIhIyhQkREREJGUKEiIiIpIyBQkRERFJmYKEiIiIpExB\nQkRERFKmICEiIiIp+x++/2QrPal/eAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f13660002d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history[:, 1], label=\"without bn\")\n",
    "plt.plot(history_bn[:, 1], label=\"bn\")\n",
    "plt.legend(loc=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}