PatWie/TheNumericsOfGANs.ipynb

## TheNumericsOfGANs.ipynb
{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Consensus Optimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook contains the code for the toy experiment in the paper [The Numerics of GANs](https://arxiv.org/abs/1705.10461)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib import slim\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "from scipy import stats\n",
    "from matplotlib import pyplot as plt\n",
    "import sys, os\n",
    "from tqdm import tqdm_notebook\n",
    "tf.reset_default_graph()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def kde(mu, tau, bbox=[-5, 5, -5, 5], save_file=\"\", xlabel=\"\", ylabel=\"\", cmap='Blues'):\n",
    "    values = np.vstack([mu, tau])\n",
    "    kernel = sp.stats.gaussian_kde(values)\n",
    "\n",
    "    fig, ax = plt.subplots()\n",
    "    ax.axis(bbox)\n",
    "    ax.set_aspect(abs(bbox[1]-bbox[0])/abs(bbox[3]-bbox[2]))\n",
    "    ax.set_xlabel(xlabel)\n",
    "    ax.set_ylabel(ylabel)\n",
    "\n",
    "    xx, yy = np.mgrid[bbox[0]:bbox[1]:300j, bbox[2]:bbox[3]:300j]\n",
    "    positions = np.vstack([xx.ravel(), yy.ravel()])\n",
    "    f = np.reshape(kernel(positions).T, xx.shape)\n",
    "    cfset = ax.contourf(xx, yy, f, cmap=cmap)\n",
    "\n",
    "    if save_file != \"\":\n",
    "        plt.savefig(save_file, bbox_inches='tight')\n",
    "        plt.close(fig)\n",
    "    else:\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "learning_rate = 1e-4\n",
    "reg_param = 10.\n",
    "batch_size = 128\n",
    "z_dim = 64\n",
    "sigma = 0.1\n",
    "method = 'conopt'\n",
    "divergence = 'standard'\n",
    "outdir = os.path.join('gifs', method)\n",
    "niter = 10000\n",
    "n_save = 500\n",
    "bbox = [-2, 2, -2, 2]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Target distribution\n",
    "mus = np.vstack([np.cos(2*np.pi*k/8), np.sin(2*np.pi*k/8)] for k in range(batch_size))\n",
    "x_real = mus + sigma*tf.random_normal([batch_size, 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Model\n",
    "def generator_func(z):\n",
    "    net = slim.fully_connected(z, 256)\n",
    "    net = slim.fully_connected(net, 256)\n",
    "    net = slim.fully_connected(net, 256)\n",
    "    net = slim.fully_connected(net, 256)\n",
    "    x = slim.fully_connected(net, 2, activation_fn=None)\n",
    "    return x\n",
    "        \n",
    "\n",
    "def discriminator_func(x):\n",
    "    # Network\n",
    "    net = slim.fully_connected(x, 256)\n",
    "    net = slim.fully_connected(net, 256)\n",
    "    net = slim.fully_connected(net, 256)\n",
    "    net = slim.fully_connected(net, 256)\n",
    "    logits = slim.fully_connected(net, 1, activation_fn=None)\n",
    "    out = tf.squeeze(logits, -1)\n",
    "\n",
    "    return out\n",
    "\n",
    "generator = tf.make_template('generator', generator_func)\n",
    "discriminator = tf.make_template('discriminator', discriminator_func)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "z = tf.random_normal([batch_size, z_dim])\n",
    "x_fake = generator(z)\n",
    "d_out_real = discriminator(x_real)\n",
    "d_out_fake = discriminator(x_fake)\n",
    "\n",
    "# Loss\n",
    "if divergence == 'standard':\n",
    "    d_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(\n",
    "        logits=d_out_real, labels=tf.ones_like(d_out_real)\n",
    "    ))\n",
    "    d_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(\n",
    "        logits=d_out_fake, labels=tf.zeros_like(d_out_fake)\n",
    "    ))\n",
    "    d_loss = d_loss_real + d_loss_fake\n",
    "\n",
    "    g_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(\n",
    "        logits=d_out_fake, labels=tf.ones_like(d_out_fake)\n",
    "    ))\n",
    "elif divergence == 'JS':\n",
    "    d_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(\n",
    "        logits=d_out_real, labels=tf.ones_like(d_out_real)\n",
    "    ))\n",
    "    d_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(\n",
    "        logits=d_out_fake, labels=tf.zeros_like(d_out_fake)\n",
    "    ))\n",
    "    d_loss = d_loss_real + d_loss_fake\n",
    "\n",
    "    g_loss = -d_loss\n",
    "elif divergence == 'indicator':\n",
    "    d_loss = tf.reduce_mean(d_out_real - d_out_fake)\n",
    "    g_loss = -d_loss \n",
    "else:\n",
    "    raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "g_vars =  tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='generator')\n",
    "d_vars =  tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='discriminator')\n",
    "optimizer = tf.train.RMSPropOptimizer(learning_rate)\n",
    "# Compute gradients\n",
    "d_grads = tf.gradients(d_loss, d_vars)\n",
    "g_grads = tf.gradients(g_loss, g_vars)\n",
    "# Merge variable and gradient lists\n",
    "variables = d_vars + g_vars\n",
    "grads = d_grads + g_grads\n",
    "\n",
    "    \n",
    "if method == 'simga':\n",
    "    apply_vec = list(zip(grads, variables))\n",
    "elif method == 'conopt':\n",
    "    # Reguliarizer\n",
    "    reg = 0.5 * sum(\n",
    "        tf.reduce_sum(tf.square(g)) for g in grads\n",
    "    )\n",
    "    # Jacobian times gradiant\n",
    "    Jgrads = tf.gradients(reg, variables)\n",
    "    # Gradient updates\n",
    "    apply_vec = [\n",
    "         (g + reg_param * Jg, v)\n",
    "         for (g, Jg, v) in zip(grads, Jgrads, variables) if Jg is not None\n",
    "    ]\n",
    "else:\n",
    "    raise NotImplementedError\n",
    "\n",
    "with tf.control_dependencies([g for (g, v) in apply_vec]):\n",
    "    train_op = optimizer.apply_gradients(apply_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "sess.run(tf.global_variables_initializer())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "if not os.path.exists(outdir):\n",
    "    os.makedirs(outdir)\n",
    "ztest = [np.random.randn(batch_size, z_dim) for i in range(10)]\n",
    "progress  = tqdm_notebook(range(niter))\n",
    "for i in progress:\n",
    "    sess.run(train_op)\n",
    "    d_loss_out, g_loss_out = sess.run([d_loss, g_loss])\n",
    "    progress.set_description('d_loss = %.4f, g_loss =%.4f' % (d_loss_out, g_loss_out))\n",
    "    if i % n_save == 0:\n",
    "        x_out = np.concatenate([sess.run(x_fake, feed_dict={z: zt}) for zt in ztest], axis=0)\n",
    "        kde(x_out[:, 0], x_out[:, 1], bbox=bbox, save_file=os.path.join(outdir,'%d.png' % i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQoAAAD8CAYAAACPd+p5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHGNJREFUeJztnXuoZld5xp83aYbgjJDaeOI0MxMlnMZJ0jiaIcYLJdYL\nySAGRSH+oa0WhtoICpbWVrBFKAgF/xCtMUWxAdEKGg3JaDoRSxQanZkwxpmcTGcSLDPj1KMJxEw0\nHUbf/vHtfbLPPuu211r7/vzgcL7L2nvtb+21n/Wud11eUVUQQoiLC/q+AELI8KFQEEK8UCgIIV4o\nFIQQLxQKQogXCgUhxEuyUIjIdhH5nog8KiJHReRDhjQiIp8WkRMi8oiIvCo1X0JId/xehnOcB/AR\nVX1YRF4I4JCI7FfVRytpbgGwXPy9GsDniv+EkBGQbFGo6hlVfbh4/QyAFQCX15LdCuAuXfAQgEtE\nZGtq3oSQbshhUawhIi8F8EoAP6x9dTmAk5X3p4rPzhjOsRfAXgDY/IIXXP/y5StzXiIhpMKhH//k\nl6r6Yl+6bEIhIlsAfB3Ah1X1V7HnUdU7AdwJALt3XacH9t+b6QoJIXUuWLrif4LS5chMRC7CQiS+\nrKrfMCQ5DWB75f224jNCyAjIMeohAL4AYEVVP2VJdg+A9xajHzcCeFpVN3Q7CCHDJEfX43UA3gPg\nJyJyuPjs7wHsAABVvQPAPgB7AJwA8GsA78uQLyGkI5KFQlV/AEA8aRTA7al5EUL6gTMzCSFeKBSE\nEC8UCkKIFwoFIcQLhYIQ4oVCQQjxQqEghHihUBBCvFAoCCFeKBSEEC8UCkKIFwoFIcQLhYIQ4oVC\nQQjxQqEghHihUBBCvFAoCCFeKBSEEC+5duH+ooisisgRy/c3icjTInK4+Pt4jnwJId2QK67HlwB8\nBsBdjjTfV9W3ZsqPENIhWSwKVX0QwFM5zkUIGR5d+iheW0Qy/7aIXNNhvoSQRLLGHnXwMIAdqnpW\nRPYA+CYWkc03UI09umNbPdYxGQp68rHgtLL95S1eCemCToSiGotUVfeJyL+IyKWq+ktD2nWxR7u4\nPhJGE3GwHUfRGCedCIWIvATAz1VVReQGLLo8T3aRN0kjVhxCzkfRGA9ZhEJEvgLgJgCXisgpAP8A\n4CJgLaTgOwF8QETOA/gNgNuK6GFkoOQWCFceFIzhk0UoVPXdnu8/g8XwKRkBXYhEPT+KxbDpyplJ\nRkCMQOixw9bv5KpdjfOmYAwTCgUBEC4SLmHwpQ0RDloXw4RCQYJEoolA+M7hEwyKxfCgUMyYrgTC\ndk6XYFAshgWFYqb4RCJEIHTFuAZwHbLzWmceFItxQKGYISkiESIOtvQm0fBZFxSLYcD9KGZGrEjo\nypHGItHkHE5x6ni4lmyEFsWMiBEJnzicO/q49btN11xpzmfliNW6cFkWAIdP+4JCMQNinZY2kXCJ\ngy1dXTRixAJgV6Qv2PWYMHrysawice7o48EiYTo2JA/b9az7PvB3kXzQopgI0Ss7G4iEjTMrq+ve\nb925ZExXnqNqXdgsixC4wKw7KBQDp82WM0Uk6uLg+q4uHOeOPu4VC18XpE5oOVFQ4qBQDIyuTOrQ\niVR1kXAJhI0zK6udiEUIpvKlePihj2IgdNnvdg2BVskhEq5j6+c3WTNtzAzdkEdR9vR72KFF0TOd\nL+nOLBLHV5+15rW8tHnDOWz+Cxeha0RywGFYM7QoeqLrFkyPHQ4WiTomkTi++qxTJMo0vnOFWBVr\n33VgXazlRQtjHbQoeqBpBezyAQH88yR8AlFP67MsQvwVa98llEWMRcJ5GwsoFB3T14pNa14NrYkm\nIlE9pi4WIdcVO2xqPaehXEP3yADm3R1h16MjQkxZV/egDZrOl0ihLjC+LkhXlGUetFp2xl2RrmKP\nioh8WkROFEGAXpUj36nQtUDEEmNNpJC6CK1xfgH3Ya5i0VXs0VuwCPizDODVAD5X/J8FrsrlrZgN\nH5ZQcz3EmkgZDs1FeZ2+39WknLzn4nqTDeTahftBEXmpI8mtAO4qtuh/SEQuEZGtqnomR/5DJlYk\nYltT03Hlg9FmC33o7HNrr6/fcrExjc9XUXdqVsl57fVzcZ8MP135KC4HcLLy/lTx2QZEZK+IHBSR\ng794ctxxj2NEIse+D12cMwdDsFiA+H0y5sTgRj3mEFKwq30fusRmRVRpOvIRS2xZNV36Pieroiuh\nOA1ge+X9tuKzyWKzJtre9wEYhnB0SZMRE19Z2XwicxeLrroe9wB4bzH6cSOAp6fsn0gViZR9H3Ic\nbyO3RWCazt30oU/9nbZzDLGr1iddxR7dB2APgBMAfg3gfTnyHTtt7ftQP1eshbF151I2P0JOkQkR\nB9t1u/bK8O3CNWeroqvYowrg9hx5jZW6NdHFvg/V8+bqjiwvbe58PkWJTyBCRK2axrf03UQbS9/H\nAGdmZiZ2Qo5pDkPT1tx1TKyJbhKfJtbB8tJmY/qmq0h9llbsPhkpi9TWpZv4RCwKRQf4rIncE51y\ni4WJUgBsouH6ziUSpha9SXcshqbTyec4ZDq44dG5kbrvg+1htO394DKvN11zpfEhKc9ju7YmFkZu\nS8KEr2tkul7XXhltLFAbG7QoMhK0MtRhysbs++D6Pqa1dfXRYzadaXJ8Ez9KrEiUaXzp+lqkNlQo\nFC3jMlNz7/uQC59YNBUM3zGbrrkyi7O1aRn4VrRWCfFVTNlPwa7HQOhr3wcb5YNrE7NU66KaR1Ns\nllcMrjJzrj2Z2egHLYqJk+rsy9Xat33OXAxl/cnQoEXRIaFDbSndiJxWRZXqgx3bf29LHPqa1zEn\nKBSZaNo/HbOzbKjWgIkcy98Jux6t0sZ4+6Gzz62r/FNnTKI0ZWhRkDWsO19zgdTsoVCMjJB9H1zY\nWmjXhKIUAWkyUakNQRrSPhljhkIxMeqVPmQYM3bWYe7ZirLzWqNY1GeM1le19rlQbS7QR9ETrr53\nly3c0KYmD+16yAIKxYTwCcxYHIMmsahfe91SihXXGAtsjrDrMVBymNO+Su9rvUNnHtpGd6w7WAeM\nBtm6IS5yd0HGIqxdQKHoEFflN+0m1aTi+1rGeqV3Oi8bTk1OTW8Vmlp5+XwVQHiZ5dgjY06w65GB\nNhcD+Uxq174PTZGrdvWyfqFJnr4uCODeKyO2vIIDK010YViukII3i8ixImTgRw3f3yQiT4vI4eLv\n4znyHTshlR5YX/HrfyZirIm+FzjZRCrGX1ElpLxM52C3Yz3JQiEiFwL4LBZhA68G8G4RudqQ9Puq\nuqv4+0RqvlMlxfw1Leceg0hUCRWLOqnl1pQhlVkX5LAobgBwQlWfUNVzAL6KRQjBWZBqapparpiK\nazpmbCJREiIWphWoTffKsKWnNbGRHM5MU7hAUwDi14rII1gE/vlrVT1qOpmI7AWwFwB2bDNGHZwF\nvu3n6unq5K7sObajbyKqctWuDU5OkzPYtH1fvUxcO2/XMZVb07kdU9y+v6tRj4cB7FDVsyKyB8A3\nsYhsvoExhRTM5biy7VUJxFkXoZXdZ03kruzV84WUXYpYVGljNIMb1zTHGy5QVX+lqmeL1/sAXCQi\nl2bIuzd8Fb2PnZptG8IMQSRM5w/Jo4mDM9WKymmFTW30I4dQHACwLCIvE5FNAG7DIoTgGiLyEhGR\n4vUNRb5PZsi7F9oQiZRK6hKIIYpEPS9ffk0cnDGC0daOW1MSi+Suh6qeF5EPArgfwIUAvqiqR0Xk\nL4vv7wDwTgAfEJHzAH4D4LYietjoaPPm+0xoU3obtgfJJRJ99qtl+8udZWvrhgDmVaf1smkjkPOc\nuh8y5Od1967r9MD+e/u+jDWCtuN3WBNNpiSbBCO0co9NJKrEWmtt7pmROtV9KGVr4oKlKw6p6m5f\nOk7hzoSvu9G0Ise0eLHTsodUkWMsC8BtXaTiCwDksyymMArCKdyBuCqvz4roYoeoKYhESYzPYu07\ni18mFd899DYUI/dXUCgCSBGJtnE9GL61G0MUiZIUsQDa2dfCJ/pTFgsKhYcYkRi6FQEMWyRKQsRi\naNaFHjvsbjxGKhYUCgexIuHj3NHHjX9NmLpIlMTOtVj3/cCsizGKBYUighiRCBGEUMGYi0iU5BCL\ntnAJxpTEgqMeFmw3sqlIxAT6ccW8dDFFkSjxjYZ4jw/cMct1v1z3xDYyMpW5FrQoDDSpkLYWJaY7\nUT8+J2MWiZJUB6eLkPvlswptdcHauIzIqqBQNCBkanaqQMQyhVYrhFjB83ULm+ITjA2fjVws2PWo\n0aTLUa8QtooTEiHbtMIxtgtSZwrWRCi2CVk2UkW9PL5+n3yTtMYGhSKSEJEIEYhqWm7uGkaqv6LE\nJRKuexcq6nWxsPkrxjBzk12PAOotVG6RcB2T2uINvQL2hcv68907W5qQe9XH9gM5oFBUyNVfjBGJ\nHMfOiSYCGDLaESIQIcfUxSJ04t3QfRUUCg9NrQlTZTu++qz1z4Sv8pE0Qu5ZE5qKxRitCvooEggV\nCRfl901iTYQ6ytjtiMd130z3auo+JloUBammX4xIuNKyC9IOucTdl2ZqViCFogFNFnrljIEJTK/i\nDZW2xD2k+zFkPwWFwoGrL+l6cGNFglZFOj6fUpV6+cbcN9c9m5K4dxVSUETk08X3j4jIq3LkOxT4\nQJOp01VIwVuwiOOxjEVwn8+l5jtWDp19DofOPtf3ZcySNqzA1GPHQo5Rj7WQggAgImVIwUcraW4F\ncFex8/ZDInKJiGxV1TMZ8h8NIQJxfPXZbNHJSTr1e3b9lot7upJ+ydH1MIUUrMcCDEkDYBFSUEQO\nisjBXzz5VIbLI2QjfXQXu9j1rC0G58xU1TtVdbeq7n7xH7yo78vpHFoTZIjk6Hp4QwoGphktW3cu\nBbVQczVbyfjpJKRg8f69xejHjQCeHoN/wrXHQxsh6HIz5HH5vgmdRVkV91ShH/Oy82ShUNXzAMqQ\ngisAvlaGFCzDCgLYB+AJACcA/CuAv0rNtw9CbzS7D9Pi+i0XO0ViDvc7i49CVfep6h+p6pWq+k/F\nZ3cUcUehC24vvv9jVT2YI9+cpK6LqLdQMZWnfsyU1w70gcsKnMPDnsLgnJljwtf9aFL5WFHHyVzE\nnULhoe6ncHU/TJUkRABMaernGoNPpEty+F9SrUBf+uo9q9cbk/9ryKt9ucy8QswWa5uuuXLdjD/T\nCAithf4I3aa/ZHlpc9BMyxBxnxK0KCKotw711j61wuSscBz5WE/IvVpe2mwVd9t3U7cAaVEE0HRn\nZyB8boXpuDohlW4qgWaGRKgl6BOJkG7H0KFFUcPWT/T5KkwP89adS42sg7ZM16lZFbl/T0q5+44N\nHVIfsn8CoEWRRL3/W4pFfZViSkXMZcKOYUv4rqj7lYDmFqDtnvru1xitCYAWhZFQqwIwtxh99U/H\nuGlrU2IizIcSYgG60vi6HDbGIOC0KBpi8leYPOs266IJuQVn7FZFV12ophag6T6ZRGKs1gRAi8KK\n64EKtSyARSVqw8KwDfn5WtWx+it81900ynyue5IqEmMRbgqFg1xiATwvGCEVNFVcpiYWsSLRJrZ7\nNDVLooRdDw+uSVi2bgjg3qQkm4PSEd/DN1w6lm5Iiki0sVGM6941FYkxlH8JLYpErCZlR0uKdeXI\nZLshQxIJn5U3ZZEAKBRB+G6qSyy6FAzj5yMVixSfRIhIhDiZQ7qLMfd4bCIBUCiCiRULoFvrwvj5\nscPu1ndgYpHbcVnl3NHHg0XCh+u+TsEvUYU+igb4Fo25pno3WZyUUpHLPEyV2OW3GIrPog2RaDpE\n3aZIDKGMY6BF0ZC2LYvQSu1rGWO6In1bFjEi4epqhFoPTZmbSAAUiijaEouYSu16GMYiFnrysWiR\nMNGWQADzFAkgUShE5EUisl9Ejhf/f9+S7qci8hMROSwig9sGL4YUsWgD28MR4/3vUixC8goViVwC\nEXOOKYsEkG5RfBTAd1V1GcB3i/c23qCqu1R1d2KegyFnBcjVApoeFtNDFTIa0qZghJ6/iUj0xdQc\nlyZSnZm3AripeP1vAP4TwN8mnnNUNJ2Q1ZSQFY2mtQnnjj6+zilnmpwVsodF+dtSRTFGdFJFIrbs\nbMSMXk3BmgDSheKySnyO/wVwmSWdAnhARH4L4POqeqfthCKyF4tAxtixzRh1cHKYKnqTJc9l2nql\nzyUWQPe+ixCRSBGIelpf2dmYepejxNv1EJEHROSI4e/WaroiALFaTvN6Vd2FRVTz20XkT2z5jTGk\nYO4KERsX03Rcjm5Il9jmfISIxJmV1axlR57HKxSq+iZVvdbw9y0APxeRrQBQ/DeWtqqeLv6vArgb\niwjopAVMD8tYxCJ0jkSqBdYFU7ImgPSuxz0A/gzAJ4v/36onEJHNAC5Q1WeK128B8InEfCdDvdKb\nKrxvV2jT3o5nVlYb76vQ576bKbMt2yqz0O7HHEgd9fgkgDeLyHEAbyreQ0T+UET2FWkuA/ADEfkx\ngB8BuE9Vv5OY72hxVfyYCl+m8aULsSqA7i0L3/TyOjmEtUwTkq5KG6tRx0KSRaGqTwJ4o+HznwHY\nU7x+AsArUvIZAzExQXw0rcjHV59d11L6WkjbMvXqg+tcqt6yqPi6HLEi4SLGEpsDXOvRI66x/9gK\n7xOLpvTlu4hpvWPKrF5eOZiafwLgFO7B0JUzLrQL0ichcyXq5ZVqSRA3tChGxqGzz617f/2Wizek\nadpKunbK6pJY0fKJRFlmprIqj2fYRzcUihFRF4nyM9sDUBLizW8qFimWyIZJX55zhfgmTNTLK6Ss\niBkKBVnDtpdF7u7JELs7xA2FgmxgSA9yn4u9fMxhMVgJhWKm5JpMlGtbOTJsKBQtk2MFqYuu+9yx\nm+tUCRWOVGvi+i0Xr/NTpJZViA9nikOjAIWiV6rBckOC5PbliMtt/lfPZxONXHmGlBlHPPxQKDIS\nMjszdJPd5aXNvc8NCH1YXQLnm+wVIho+cpYVZ2WaoVBMkBwtpEskYvZ7AJqJRle4ysoZz2NGjkyA\nMzN7p1oZ6w9SzANvOqZJK+nbdzJlBmnKfhFVTL8nR1m5ymkIE9L6hELRAfXWx1XpTGIR+hCkWhI+\ngcg1zbyt6eqhZRWSLqYbNFVHJsCuR3ZiVpFWnZo2yopd74v7KnyoNRFjRYT6BXLtl1HF5fyNEcwm\n1sTcuh0AhaI3XE7NstKaHoQmD4Gr8oe2mKkiUaZtSyzK86SeowrnfWyEXY+O8LVCpsoZ+xBt3bmU\nRSRsxC7nbouUcqpTL5tQa2LK3Q6AQtEKtkrj81XYxCL0QQgRiCYikXtjGNOxPmsg9JrL3+4rK1c6\nn0jMGXY9eqbeBSkra91nkDq+P0ZzunrN1dc+f07TsjKVjUkk5mpNAOkhBd8lIkdF5HciYo0AJiI3\ni8gxETkhIq5oYpMh1KoAzJUyx4Ndtsa5RMJlTRw6+9y6v1Rc15zrd9nO0UQk5kKqRXEEwDsAfN6W\nQEQuBPBZAG8GcArAARG5R1UfTcx7tJjWf5icm01aUdMxXdFkn4zcm8TUf6+vnHzl01Qk5mBNAOmb\n664AgIi4kt0A4ESxyS5E5KtYhCKcvFA0DTfoGgkZY9chlpTfGnuszR9BkVjQhTPzcgAnK+9PFZ8Z\nEZG9InJQRA7+4smnWr+4tnFVJls3pA8nWldC1GQ2ZBe4ypsi8Txei0JEHgDwEsNXHyuihWWliEt6\nJwDs3nWdLUThqIgJZFytvLm2netyQ5rYla5B8T4Tf1PQcvGZ+yTqeIVCVd+UmMdpANsr77cVn82K\nlKjnuSwM3wNWnyFan/1oW6XZxfJ3a6uf2foKEYi5WRNAN12PAwCWReRlIrIJwG1YhCKcHb5uSJet\nWB/dG1+3w2ZNdHGtoeU/R5EA0odH3y4ipwC8BsB9InJ/8flaSEFVPQ/ggwDuB7AC4GuqejTtsseL\nr6KVFbYL0YgZlu16k5e2RaKJQMxVJABAVIfrBti96zo9sP/evi+jFZouHGtzO70uQvfFWBN9dCuM\nx01YIC5YuuKQqlrnQJVwZmZPNF1lmmJhNBUZn68CCN9VKmR/jBiR6KqbNmWRaAKFokfKSpg7uPGG\nfIqHyiYYtsleIWJRUhUNV/ckZDjUJRIUiH6gUAyAIQhGyF6eqXtApCzE6sRnQ3GwwtWjA6J0mLVd\nYa2LmwJXs8YwVJHoqszHDi2KgVKvuLmtDddEr/pqVtdK1tgduPsQCYpBPBSKkeCr5DFC0kQsAPOC\nq7aWdJfX1wQKQXtQKCaC6SEJEY9QsQDcguGj6WQqTn4aFhSKCRPqJG0iFkD48veYJd3l9TiPo0B0\nDoViBoQIhm9xWs7l77QixgeFYkb4Jnn5hk+BfCtZbXlbv6dA9AqFYmakWBdA8yXeOZZ0UyT6h0Ix\nU1Ksi7U0iWsx2NUYDxSKGRMUfT1AMBrnGzjsSZEYDhSKmdNkZARIEwwKxHihUBAA4atZTQ/7hk2C\nE2ZTUiSGCYWCrBG7OC3HNGsKxLDhojCygS4fWi7IGge0KIiR6sPbxvJ3isO4SBIKEXkXgH8EsBPA\nDap60JLupwCeAfBbAOdDtt4iwyGXaFAcxkvrIQUrvEFVf5mYH+kZPuzzpIuQgoSQkdOVM1MBPCAi\nh0Rkb0d5EkIy0VVIwder6mkRWQKwX0QeU9UHLfntBbAXAHZss4YoJYR0SBchBaGqp4v/qyJyNxYR\nzo1CMcXYo4SMnda7HiKyWUReWL4G8BYsnKCEkJHQekhBAJcB+IGI/BjAjwDcp6rfScmXENItqaMe\ndwO42/D5zwDsKV4/AeAVKfkQQvqFU7gJIV4oFIQQLxQKQogXCgUhxAuFghDihUJBCPFCoSCEeKFQ\nEEK8UCgIIV4oFIQQLxQKQogXCgUhxAuFghDihUJBCPFCoSCEeKFQEEK8UCgIIV4oFIQQLxQKQoiX\n1M11/1lEHhORR0TkbhG5xJLuZhE5JiInROSjKXkSQron1aLYD+BaVb0OwH8D+Lt6AhG5EMBnAdwC\n4GoA7xaRqxPzJYR0SJJQqOp/qOr54u1DALYZkt0A4ISqPqGq5wB8FcCtKfkSQrolNZp5lfcD+HfD\n55cDOFl5fwrAq20nqYYUBPB/FyxdMcVgQZcCmGJk96n+LmC6v+2qkERZYo+KyMcAnAfw5SZXaKIa\nUlBEDqrq7tRzDg3+rvEx1d8mIgdD0iXHHhWRPwfwVgBvVFVTrNDTALZX3m8rPiOEjITUUY+bAfwN\ngLep6q8tyQ4AWBaRl4nIJgC3AbgnJV9CSLekjnp8BsALAewXkcMicgewPvZo4ez8IID7AawA+Jqq\nHg08/52J1zdU+LvGx1R/W9DvEnNvgRBCnoczMwkhXigUhBAvgxaK0CniY0RE3iUiR0XkdyIy+mG3\nqU7TF5EvisiqiExqPo+IbBeR74nIo0U9/JAr/aCFAgFTxEfMEQDvAPBg3xeSysSn6X8JwM19X0QL\nnAfwEVW9GsCNAG533bNBC0XgFPFRoqorqnqs7+vIxGSn6avqgwCe6vs6cqOqZ1T14eL1M1iMSF5u\nSz9ooajxfgDf7vsiiBHTNH1rpSPDQkReCuCVAH5oS5NzrUcUXU8R75KQ30ZIn4jIFgBfB/BhVf2V\nLV3vQpFhivhg8f22CcFp+iNERC7CQiS+rKrfcKUddNcjcIo46R9O0x8ZIiIAvgBgRVU/5Us/aKGA\nZYr4FBCRt4vIKQCvAXCfiNzf9zXFkjhNf9CIyFcA/BeAq0TklIj8Rd/XlInXAXgPgD8tnq3DIrLH\nlphTuAkhXoZuURBCBgCFghDihUJBCPFCoSCEeKFQEEK8UCgIIV4oFIQQL/8PMFBv4ydSsUoAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f10250dd438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQoAAAD8CAYAAACPd+p5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGjVJREFUeJztnX/MnWV5x7/XGE0YtqFd5wvrj9Dk7UvKhmJpClb/wCkG\nCFmn0QWWqJsmjQqJJi4bm4nbnyRLTMZAWJ1ESQhIomgTqvxYDJVoFeiAUgrv3rRM3tqVlDa0i2Sk\n7tof55z29Ol9P/fP5/f3kzR9zzn3ee7nPM99f5/ruu4fl6gqCCGkjN9p+gQIIe2HQkEIcUKhIIQ4\noVAQQpxQKAghTigUhBAnyUIhImtE5Cci8rKI7BORLxnKiIjcKSILIvKiiGxMrZcQUh+/m+EYpwB8\nRVX3iMhSAM+JyBOq+vJUmRsArB//uxrAPeP/CSEdINmiUNXDqrpn/PdJAPsBrCoU2wrgfh2xG8BF\nInJJat2EkHrIYVGcRkQuBfA+AL8ofLQKwOtTrxfH7x02HGMbgG0A8HsXXnjV+rnLcp4iIWSKF/5j\nz1FV/QNXuWxCISLvAvA9AF9W1ROxx1HV7QC2A8CVG6/Sx5/anekMCSFFZpYt+S+fcllGPUTkfIxE\n4gFV/b6hyCEAa6Zerx6/RwjpADlGPQTAtwDsV9WvW4rtAPDp8ejHNQDeUtVz3A5CSDvJ4Xp8AMCn\nAOwVkefH7/09gLUAoKr3AtgJ4EYACwB+A+CvMtRLCKmJZKFQ1acBiKOMArg1tS5CSDNwZiYhxAmF\nghDihEJBCHFCoSCEOKFQEEKcUCgIIU4oFIQQJxQKQogTCgUhxAmFghDihEJBCHFCoSCEOKFQEEKc\nUCgIIU4oFIQQJxQKQogTCgUhxAmFghDiJNcu3PeJyBsi8pLl82tF5C0ReX7872s56iWE1EOuvB7f\nBnAXgPtLyvxUVW/KVB8hpEayWBSqugvAsRzHIoS0jzpjFFvGmcx/JCJ/VGO9hJBEsuYeLWEPgLWq\n+j8iciOAH2CU2fwcpnOPrl6ztqbTI4SUUYtQTOciVdWdIvINEVmpqkcNZc/KPVrH+ZE4Fo+9bXx/\n9YoLaj4TUjW1CIWIXAzgiKqqiGzGyOV5s466SR5souBTlsLRfbIIhYg8COBaACtFZBHAPwA4Hzid\nUvATAL4gIqcAvA3g5nH2MNJiQsTB9zgUjW6SRShU9RbH53dhNHxKOkAugSg7NgWjW3BmJjmLKkWi\nWE9ddZF0KBQEQHMdl2LRDeoaHiUtJqSz7j96wl1oig0rl3nXT3ekvVAoBo6PSISKg+27LtFYPPY2\nxaKl0PUYMC6R2H/0RJJImI7ngq5IO6FFMUByWhE/+9XJs15vWbvU67hl1gUti/ZBoRgYPlZEGUVh\ncH1uE479R09QLDoEXY8BkSISP/vVSadIhH7PJUp0Q9oDLYqBECsSMeJQdpyiheFyRTgi0g4oFD0n\nJR5hE4m9B91bj1yxboX1mCZ3hK5Iu6FQdICyzj7deWJM9RCR8BEIU9miaNjEwgWti+agULSImI6e\n4sdXJRKm7/qIhcuqmOArnCQfFIqGaSpgl0skFuaPnPV6dm7GWC63WNjwvZ4UlDAoFA3RZEQ/VSSK\n4mD7rCgadYmFD1z6HgaHR2um6VWTVYqET1lfFybnjFAXTd+TLkChqJGmG2NI5yt26IX5I0EiUfa9\n4rFj51nkhoJhh65HTVS5QhMonxIdOtsyJXAZS9mwKeC3CjUXHF05FwpFxfgKROrTM/b7PhOqbJbE\n8YV54/vLZ+eMx5iOWZjiFWWYVqHmFtQinLtxBroeDZN7hWZd2ESi7DOX6+I7CzTlmk2+6/t9uiIj\n6so9KiJyp4gsjJMAbcxRb9spa2RtEAifAKapc5eJREgZk4uTa8q4D76iQbGoL/foDRgl/FkP4GoA\n94z/7y0ukXDh22FiZjjWxfGFeaMb0kY4hbycXLtw7xKRS0uKbAVw/3iL/t0icpGIXKKqh3PU3zZi\nRSJ2deY0vsIR++T2sRRSiZ3i7fObyo7rs0BtqGJRVzBzFYDXp14vjt87Ryj6nFKw6hWaxWPZOkXK\nYq/c2IKaLrGIvWY+16fMuhiqWLRu1KPrKQVt1kSKSJQtsCqjTn+/CopL03P/HtvSd6Ce2aFdoi6h\nOARgzdTr1eP3Bk3Mk930WYh4pLJ8ds7b/cgVn6ha8EKXvg/RqqhLKHYAuE1EHsIoiPlWH+MTIdZE\nzhWak+/VKRhtI+Taxbg6QyfX8OiDAH4O4DIRWRSRz4nI50Xk8+MiOwEcALAA4JsAvpij3q6Sexl3\nzmP44GMp1DXasffgseDfbfuO6b7YXMahDZlKm3MFX7nxKn38qd1Nn4YXvtZEiEiUTVCyLeeeEGtd\nhC4EM7khNpGwnXPMueYUxWL9JsvC5IL0wf2YWbbkOVXd5CrXumDm0IhdoTkpE7L/QxWkWg5ViITt\n+vleK183ZEixCk7hrhCXNZG6jDumvAtTx3VZLz7kOAbg3kTHtVdG7PVqehZt01AoWkRsI7Z9L6d5\nPjs3E93Zc7ocNkL3yYhd+j5UKBQZiAls+aypCCGnZVHWgUPEokxccrocuQQ2RliHEtRkjKImck2Z\nLosJFJdyA/GxiivWrbB2nFQ3oilLIpRirGLIk7BoUVREiE8bst/D8YX5WtZbAKMOnbNT5z6ei8m1\nKrteOayKIUCLoqXUJQY+TDp3bCeqShxCBRaob35H36BQNECOp1YTS7inO7zrN7R1lqjpuplcNnI2\nFIqBY5ov4BNPySkEW9YuzTbKkNsS85lTMYT5FBSKnlPWoW0dYPr9KoYJTfVWtUI0N0MNaFIoOkqq\n2+G7ACqXaITU13axGCIUigYoG3qcELKcO5TYVZJ1ra5MEYsqr9uQ4fBoB/FddGVyO7qylNp0nsXf\nwwBkfdCiaAGzczPGoT4O5cVRZlXwmsZBoegJuayJYqCuycVQPi5IDpGlZeKGQlETxUZfjFPYGrwP\nPg3dJRK2SH4dEf4yMXJdN1IPjFFUREwHi3mymb4TMsdhw8pljQ/3hZ5DzlhFLmui74vDKBQtI3R1\nZpEQl6NpgShiOx8fl6kq96Erwd+qybVn5vUi8uo4ZeDths+vFZG3ROT58b+v5ai3DaQ8SWxP/rLl\n2ZPP+iYSE3zFwrbBThVCSzLEKETkPAB3A7gOo8Q+z4jIDlV9uVD0p6p6U2p9XcYUnMu1nLsPIjFh\nw8plxriFb7xict1Ct8QjdnIEMzcDWFDVAwAw3pJ/K4CiUJCK6JNITLCJRZGc+2akWhN9XvORQyhM\n6QJNCYi3iMiLGCX++WtV3Wc6WJ9TCtqIjeTbGnYOkaiqwYe4aiaxsFllAPeSqJK6gpl7AKxV1fcA\n+BcAP7AVVNXtqrpJVTf9/sqVNZ1eHKkZy6cJeZqVbQCTKhKrV1xQ6VNxcnzfOkznbfuNKRZByHfL\n7m1fRz9yWBTOdIGqemLq750i8g0RWamqRzPU3xvK9nvwacgpItGEyTyp09W5fC0LIM66YADTTQ6L\n4hkA60VknYgsAXAzRikETyMiF4uIjP/ePK73zQx1N0bsk8N3uG1iNfhuH9c1kSjW7zoHm2VRZl34\nXLdYkRiaVZFsUajqKRG5DcBjAM4DcJ+q7pukE1TVewF8AsAXROQUgLcB3KxtTlGWSFkjyr2Eukx4\nXCLRtEAUWb3igtJO5jsaMk2qtRCbk7RvgU2mFIwgJTaRUyj6JBLTuJ7Itmtc1T4Wsde5zdd4gm9K\nQc7MDCTFrMzVkMtMbqDbIgG4z69sUlYVMynL7ttQXBAKRUbqcDliF3dNaLtITIgVC6CaaddDFwsK\nRQCxLgdFIg6KRXugUHhCkWiGNoqF7Z72OZExhaJCKBJ5aJtYAPZ7axOLrlsV3LjGgxhroq6dpLuy\ndiOV2KFTX2wTtMqGV0OHTrs8ZEqLIoGmRcKHrjZME7G/xdWZy2Zx7j14rPRz073uowtCoXBge4ql\niMSk8aUuYuq7y2Gi7DfFWFe+96DsfoWIRVddELoeGSkTCVsjm7zP9Qb1EyPUtvtlckP6lFWMFkUJ\nIdaETSR8LYfcS6T7aE1MyG1VxGC6X74uZxetCgpFhYR2/hDTlqSRQ5h9jtGXeAVdjwyYOnJsQ9x7\n8FiyG9Jna2KCaxRkmhChLUuZYNoxq3i/YheRtR1aFIH4PCG401K7sd0fV16VhfkjxjKu+21qM11z\nPygUFqp4WvlCoamfkORLrrJ9dBUpFJlxdXLbU4k0R4770Xdxp1AEkBKYKgpEqGD08SnVZXJYFV1y\nPygUNVDWqGyf9f0JlYMcQdsUayLku10f/aBQJFB8arBzD5s+3/+6UgqKiNw5/vxFEdmYo15CQulz\nZ66SZKGYSil4A4DLAdwiIpcXit0AYP343zYA96TWS0jVHF+Yx/GF+aZPoxXksChOpxRU1XcATFIK\nTrMVwP06YjeAi0Tkkgx1E0JqIIdQmFIKroooA2CUUlBEnhWRZ9882u78QD4Ztkm7CLlHy2fnsHx2\nrsKz6Q6tC2a2OaVgX1YCEhJKDqFwphT0LNNbyrJq2z6jdVIPoRnPy+jzPaslpeD49afHox/XAHhL\nVQ9nqLt12BrL7NzM6X/Tr0l+ipZfVYu0hnT/koVCVU8BmKQU3A/g4UlKwUlaQQA7ARwAsADgmwC+\nmFpv1fhO5olphDENrI8rEttCFR2+eL+67rZmWWauqjsxEoPp9+6d+lsB3Jqjrqbx2cT1inUrOF7f\nYkz3Z3ZuJmimZVFcYtyOLm0H0LpgZhcxPe1T/NU++7ptxteyGJLLMYFCUUKq4sd0eIpENfgOZZfF\njmyfFY/VN7cDoFBEYbrxthhCEx2/S6sSm6Ts3kwHn8vEI/b+dsntACgU0YSKhatB+ZQh/vg+xXO7\niH20JgDumekkZG9GYNRQbHsRxDbKmBGPLmelqgrbvYkJPvuIRJ+gRZGA7WlRV4Pp+h4HbSLE6vMV\nCVv76KKA06LwoMyqsA2XllkWddF3qyImFuO6LzFWX58tiQm0KDyJ6XBDaEBdJOd9sR2rT9YEQKHI\nQlnAasvapZUKhsv96OsISOrvSr0nZfe1byIBUCiCSEllR7HIR67fE3NPXMLfl1GOIhSKQFLFIqZx\n+sQ6hiIWvr/DN9Drc08mZVzlyu5/l60JgMHM7PisBYkJdPqkqnNlz+56cLNKsUux+FwPiC5f8wm0\nKCJw3Xgf85OWRRgh513nsPEQRAKgUETTpFi4BKNv8ytC3I26fvuGlcsGIxIAhSKJHGIRS4pYdMmq\nyB2TyIHPfe2TSACMUXSaiVjYLJOymEUX4hU+ItE2gQD6JxIALYpkmrQqJpRZF120LBaPve08tzrd\nDGDYIgFQKLLQhsbRF7FomxUBUCSARNdDRFYA+C6ASwG8BuDPVfW4odxrAE4C+C2AU6q6KaVeYqbM\nFemCG5JLJFzxm5AgMkViRGqM4nYA/66qd4xzjt4O4G8tZT+kqu3O6FMRtrkVrgZdtvS5bPGSz5yL\nIpNO2lSD93E1ygiZl+KK7YTSd5EA0l2PrQC+M/77OwD+LPF4nSV3Y3Htj7D34LHSMqaO4/M0rtsV\n8Y1H2PAZLi77bhlDHN2wkSoUM1P5Of4bgG3XUQXwpIg8JyLbyg7YpZSCVRGyiUqZYKSIhU8HTsH3\n+C6RSKXprQC6gtP1EJEnAVxs+Oir0y9UVUVELYf5oKoeEpF3A3hCRF5R1V2mgqq6HcB2ALhy41W2\n4/WW2G3+9x485r2Xgmuq9zQxYmF6ysYcxyYSbencQ7EmAA+hUNWP2D4TkSMicomqHh5nJ3/DcoxD\n4//fEJFHMMqAbhQKEo9JLGzxihCxCCWHJRIrEjahjYnpVHmNukaq67EDwGfGf38GwA+LBUTkQhFZ\nOvkbwEcBvJRYbysJecKYGnyOpEGmY9g6V1uneseIhCtmExPTIWdIFYo7AFwnIv8J4CPj1xCRPxSR\nSeawGQBPi8gLAH4J4FFV/XFivSQTbROLUJFwCUBqeRtDcjuAxOFRVX0TwIcN7/8awI3jvw8AeG9K\nPUPHlurOlmsixAUB2mNix4gEqQfOzGwJtkZflg8zJFcm4J69Wfe06GLdIaSKRIiLRrgorPMszB8x\nWhYhoyBFqhALk8USO8uyTCRM4hlieREzFIqBETNrMwcx4hMiEi7La4iJhXNC14O0khA3wMcFC3XT\nJrQt2NsUtCh6jM20TrUqyjpxldZKalzCZFnQ/fCDQjFQfMQiJrhn+k6oeOS2JkLrZuKmc6FQtISY\nRLmpFFdRVhX1nz5urDhxKLRZKBSZ8c1+7rtl/+zcTOlTM0eQrs5hwWJdVYsUyQOFoiZ88n3YmIhB\nUTBcIpHb9w59qvvUT4HoBhSKFuFyP5oY4ksx+SffZbCw+3B4tEFM/npbOlWuNRG5j1U1xXvShqnt\nbYAWRQXY4hS+7keOwGaK4Ljqdo00lM2EBJoTQ9N5tUWY2w4tioaxjQKkNOCqRGJh/oj35Kaycjmt\niybcsaGtHAUoFLVjMmXLxCKk04eWL1I2PTpmvkLZ93IGRmfnZkoFw/Y5rQl/6HpUhO8w6YSy4dIm\nG3SOCU11rbVIrYMTrezQomgAW4CsyYZqesKXicTxhXnjPxumY1Ux3JrzWAxknoFCUSFlvmwbxcIH\nlyC4Pk8lh1iYjuF73YcYnwAoFK1ky9qlrRSMEAEwlc21LiMlFuMrErQmziY1peAnAfwjgA0ANqvq\ns5Zy1wP4ZwDnAfg3Vb0jpd4uURarmDRG25DpdAP2mcGYc5FX7sVWVTDd6WOzqhE/UoOZLwH4OIB/\ntRUQkfMA3A3gOgCLAJ4RkR2q+nJi3Z3BFdj0mV+Rw8LwXV+Si+ML81g+O1dLXbFiEGJNDNXtABJd\nD1Xdr6qvOoptBrCgqgdU9R0AD2GUipBMUZepGys4VcYdmsDm3tHlMFNHjGIVgNenXi+O3zPS15SC\nPk+jDSuXsaEWmHTonDGbmGMN2ZoAElMKquo5CX9S6XNKQd+5FdNiUcVWbCYXxDVtfPnsXK1Whakz\nm97zdaV8xIEibScppaAnhwCsmXq9evzeIAmdiOXbeEMFxRWvcO2D4UNsfCLkiZ/D0nBd46FbE0A9\nMzOfAbBeRNZhJBA3A/iLGuptLZOGlzNbeOx2+HVSnDmZMp8hFxQJP5JiFCLyMRFZBPB+AI+KyGPj\n90+nFFTVUwBuA/AYgP0AHlbVfWmn3Q+qboSumEexU7pGDkIshLpGO2LxiQdRJM4gqu0NA1y58Sp9\n/KndTZ9GLeS0Lkz4pusrxilM7ocrVmESiTZYEyExiKGIxMyyJc+p6iZXOS4KawnTDbMK0bDN1SjG\nKopBTVOsYiIEJsGoMi5RV7BxKCIRAoWihVQlGin7dprwFQUfa8JG3SMRFAkzFIqWU9ZwY0TEJBY+\nVgUQN63bZ+m3zZqoUyQoEOVwUViHWb3ignP++eCzeY7pqR+y34PvZjEUiW5Ai6Jn+A69+rghpklY\n050/JHP45HjTNCkSFIcwKBQ9JWauRuiMTV8LI2SEoyqRoDCkQaHoOa5l7q54BXCmo8dsilu1SFAA\n6oFCMQBixAI4d46Fr2CUjWqkigSFoRkoFAMhVCwA+3qQnHs/TOp3QYFoFgrFgIgVCyAtR2jZZCpO\no+4GFIqBESMWQJxgpAgEQJFoExSKAZJrH89YKBLdgxOuBoqrI1YxTOm7YpMi0T5oUQwYn01/gfR9\nLTii0X0oFAPHZ8etYkf3EY5Qi4Qi0W4oFCR4FmdOt4QC0Q0YoyCnqbvTUiS6Ay0KchZV7Odpq4N0\nh7pSCr4G4CSA3wI45bP1FmmW3IJBceg2lacUnOJDqtqfjD4DodjBQ4SD4tAfkoRCVfcDgIjkORvS\netj5h0ldwUwF8KSIPCci22qqkxCSibpSCn5QVQ+JyLsBPCEir6jqLkt92wBsA4DVa9Z6Hp4QUiV1\npBSEqh4a//+GiDyCUYZzo1D0OfcoIV2lctdDRC4UkaWTvwF8FKMgKCGkI1SeUhDADICnReQFAL8E\n8Kiq/jilXkJIvaSOejwC4BHD+78GcOP47wMA3ptSDyGkWTiFmxDihEJBCHFCoSCEOKFQEEKcUCgI\nIU4oFIQQJxQKQogTCgUhxAmFghDihEJBCHFCoSCEOKFQEEKcUCgIIU4oFIQQJxQKQogTCgUhxAmF\nghDihEJBCHFCoSCEOEndXPefROQVEXlRRB4RkYss5a4XkVdFZEFEbk+pkxBSP6kWxRMA/lhV3wNg\nHsDfFQuIyHkA7gZwA4DLAdwiIpcn1ksIqZEkoVDVx1X11PjlbgCrDcU2A1hQ1QOq+g6AhwBsTamX\nEFIvqdnMp/ksgO8a3l8F4PWp14sArrYdZDqlIID/nVm2pI/JglYC6GNm977+LqC/v+0yn0JZco+K\nyFcBnALwQMgZmphOKSgiz6rqptRjtg3+ru7R198mIs/6lEvOPSoifwngJgAfVlVTrtBDANZMvV49\nfo8Q0hFSRz2uB/A3AP5UVX9jKfYMgPUisk5ElgC4GcCOlHoJIfWSOupxF4ClAJ4QkedF5F7g7Nyj\n42DnbQAeA7AfwMOqus/z+NsTz6+t8Hd1j77+Nq/fJWZvgRBCzsCZmYQQJxQKQoiTVguF7xTxLiIi\nnxSRfSLyfyLS+WG3vk7TF5H7ROQNEenVfB4RWSMiPxGRl8ft8Etl5VstFPCYIt5hXgLwcQC7mj6R\nVHo+Tf/bAK5v+iQq4BSAr6jq5QCuAXBr2T1rtVB4ThHvJKq6X1Vfbfo8MtHbafqqugvAsabPIzeq\nelhV94z/PonRiOQqW/lWC0WBzwL4UdMnQYyYpulbGx1pFyJyKYD3AfiFrUzOtR5R1D1FvE58fhsh\nTSIi7wLwPQBfVtUTtnKNC0WGKeKtxfXbegSn6XcQETkfI5F4QFW/X1a21a6H5xRx0jycpt8xREQA\nfAvAflX9uqt8q4UClinifUBEPiYiiwDeD+BREXms6XOKJXGafqsRkQcB/BzAZSKyKCKfa/qcMvEB\nAJ8C8CfjvvW8iNxoK8wp3IQQJ223KAghLYBCQQhxQqEghDihUBBCnFAoCCFOKBSEECcUCkKIk/8H\nrr1ILCURUOkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0f887e5080>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Real distribution\n",
    "x_out = np.concatenate([sess.run(x_real) for i in range(10)], axis=0)\n",
    "kde(x_out[:, 0], x_out[:, 1], bbox=bbox, cmap='Reds')\n",
    "# Fake distributions\n",
    "x_out = np.concatenate([sess.run(x_fake) for i in range(10)], axis=0)\n",
    "kde(x_out[:, 0], x_out[:, 1], bbox=bbox)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "widgets": {
   "state": {
    "6a99f9ece5b341c3a3c74215c6b62699": {
     "views": [
      {
       "cell_index": 10
      }
     ]
    }
   },
   "version": "1.2.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}