ode_tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " # TensorFlow with a ODE Solver\n",
    " \n",
    " Adapted from [here](https://medium.com/r/?url=https%3A%2F%2Fwww.tensorflow.org%2Ftutorials%2Feager%2Fcustom_training_walkthrough)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The MNIST Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils import np_utils\n",
    "from keras import datasets\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "if not sys.warnoptions:\n",
    "    import warnings\n",
    "    warnings.simplefilter(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reduce_dataset(x,y, classes=(0,1)):\n",
    "    x_redu = []\n",
    "    y_redu = []\n",
    "    for i in range(len(y)):\n",
    "        if y[i] in classes:\n",
    "            x_redu.append(x[i])\n",
    "            y_redu.append(y[i])\n",
    "    return np.array(x_redu), np.array(y_redu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_val, y_val) = datasets.fashion_mnist.load_data()#tf.keras.datasets.mnist.load_data()\n",
    "\n",
    "X = np.concatenate((X_train, X_val), axis=0)\n",
    "y = np.concatenate((y_train, y_val), axis=0)\n",
    "\n",
    "X, y = reduce_dataset(X, y, (0,1,2,3,4))\n",
    "\n",
    "# subsample\n",
    "X_train, X_val, y_train, y_val = train_test_split(X, y,  \n",
    "                                                  train_size = 500, \n",
    "                                                  test_size = 500, stratify=y, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1, 2, 3, 4], dtype=uint8), array([100, 100, 100, 100, 100]))"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(y_train, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1, 2, 3, 4], dtype=uint8), array([100, 100, 100, 100, 100]))"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(y_val, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train matrix shape (500, 28, 28)\n",
      "Test matrix shape (500, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "# print the initial input shape ready for training\n",
    "print(\"Train matrix shape\", X_train.shape)\n",
    "print(\"Test matrix shape\", X_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train shape: (500, 784)\n",
      "y_train [2 4 3 0 1 0 0 3 0 1 4 0 3 0 0 4 1 3 3 1 1 1 2 2 4 2 0 4 4 3 4 1 1 2 2 0 0\n",
      " 2 1 4 3 4 0 0 2 4 0 2 0 2 1 0 3 2 3 4 4 3 3 0 2 1 2 3 1 0 4 4 4 2 1 3 0 1\n",
      " 4 4 2 2 0 1 3 4 2 3 3 2 3 4 2 2 3 0 3 1 1 2 0 4 3 0 4 2 1 0 2 1 0 1 3 3 1\n",
      " 2 0 1 4 3 4 1 2 4 3 0 1 2 2 4 0 2 3 2 1 1 1 0 1 4 2 1 2 1 2 4 1 3 1 0 4 4\n",
      " 4 2 1 3 0 3 1 4 3 2 3 4 2 2 4 4 2 3 1 3 1 1 0 0 1 0 2 4 4 4 0 1 4 1 1 1 2\n",
      " 0 2 3 2 4 3 0 1 2 1 2 0 2 4 3 1 3 1 1 3 0 0 0 2 1 2 1 0 2 1 2 2 3 2 1 3 0\n",
      " 4 0 2 1 3 2 4 4 1 3 2 3 1 4 2 3 3 1 1 1 1 0 1 1 2 4 2 0 4 4 4 4 4 0 4 3 2\n",
      " 0 4 1 4 3 4 2 3 3 0 4 0 3 1 0 1 4 4 3 1 1 1 2 2 2 1 2 4 1 1 2 4 4 2 0 0 0\n",
      " 4 3 2 2 3 1 4 1 0 3 0 4 1 2 3 0 4 2 1 3 1 3 0 0 0 1 0 4 0 0 1 3 1 2 2 3 2\n",
      " 3 2 2 2 4 3 1 0 4 2 0 4 2 3 2 2 3 0 2 4 1 3 3 2 2 3 0 3 4 0 4 2 2 4 1 3 0\n",
      " 3 0 4 4 4 3 3 0 0 1 3 3 1 4 3 0 4 0 2 4 0 2 0 4 0 3 0 0 2 3 0 0 2 3 1 4 4\n",
      " 1 3 0 4 4 1 0 3 3 1 3 3 4 1 3 4 1 0 4 0 3 3 2 0 0 4 1 3 0 2 0 3 1 1 1 2 1\n",
      " 4 0 0 0 3 2 3 2 2 1 3 4 4 1 2 3 3 3 1 3 2 3 0 4 0 2 3 4 0 1 4 0 2 1 0 4 2\n",
      " 1 0 4 1 4 4 3 4 3 0 3 0 3 4 3 4 2 2 0]\n",
      "Shape after one-hot encoding (train):  (500, 5)\n",
      "Shape after one-hot encoding (val):  (500, 5)\n"
     ]
    }
   ],
   "source": [
    "# building the input vector from the 28x28 pixels\n",
    "X_train = X_train.reshape(X_train.shape[0], 784)\n",
    "X_val = X_val.reshape(X_val.shape[0], 784)\n",
    "X_train = X_train.astype('float32')\n",
    "X_val = X_val.astype('float32')\n",
    "\n",
    "# normalizing the data to help with the training\n",
    "X_train /= 255\n",
    "X_val /= 255\n",
    "\n",
    "print('X_train shape:', X_train.shape)\n",
    "print(\"y_train\", y_train)\n",
    "\n",
    "# one-hot encoding using keras' numpy-related utilities\n",
    "n_classes = 5\n",
    "y_train = np_utils.to_categorical(y_train, n_classes)\n",
    "y_val = np_utils.to_categorical(y_val, n_classes)\n",
    "print(\"Shape after one-hot encoding (train): \", y_train.shape)\n",
    "print(\"Shape after one-hot encoding (val): \", y_val.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TensorFlow custom training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow version: 1.14.0\n",
      "Eager execution: True\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "tf.enable_eager_execution()\n",
    "\n",
    "from tensorflow import contrib\n",
    "tfe = contrib.eager\n",
    "\n",
    "from scipy.integrate import ode\n",
    "import multiprocessing as mp\n",
    "\n",
    "print(\"TensorFlow version: {}\".format(tf.__version__))\n",
    "print(\"Eager execution: {}\".format(tf.executing_eagerly()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<DatasetV1Adapter shapes: ((?, 784), (?, 5)), types: (tf.float32, tf.float32)>\n",
      "Prediction: [1 4 1 4 0 0 0 0 1 4 1 0 1 1 4 0 4 0 0 4 0 4 0 1 1 1 4 1 1 0 1 4 4 1 0 1 1\n",
      " 1 4 1 0 1 0 0 1 1 1 1 4 1 4 4 4 4 0 4 4 4 0 1 1 1 1 4 4 0 1 0 0 1 4 4 0 4\n",
      " 4 0 1 1 4 4 0 1 1 0 4 1 4 1 1 0 0 0 4 1 4 1 4 0 0 1 1 0 4 4 1 2 1 4 4 0 1\n",
      " 1 1 0 4 1 1 4 1 0 4 4 1 1 1 1 4 1 0 4 0 4 4 1 4 0 4 4 1 4 1 4 4 0 4 4 1 0\n",
      " 0 4 4 0 4 1 1 0 0 1 4 1 1 1 1 1 0 0 4 0 4 4 1 4 4 4 1 0 4 1 2 4 0 4 4 4 1\n",
      " 4 1 0 1 1 0 0 4 1 1 1 4 1 1 0 4 0 4 4 4 4 1 0 0 1 1 4 4 1 4 1 1 0 1 4 4 0\n",
      " 1 0 1 4 1 0 0 1 1 4 2 0 0 1 1 4 1 4 4 4 4 1 4 4 1 0 0 0 1 4 1 0 1 1 0 0 1\n",
      " 4 1 0 4 4 1 0 4 1 1 4 4 4 4 2 4 1 1 0 4 4 1 4 1 1 4 1 1 4 1 0 1 0 1 0 0 1\n",
      " 1 0 1 1 1 4 4 4 0 0 4 4 4 1 0 1 1 0 4 4 4 0 0 4 4 4 0 4 1 2 4 0 1 1 1 0 1\n",
      " 1 1 4 0 0 4 4 4 0 1 1 1 1 0 1 1 4 1 1 0 4 4 0 1 0 4 0 0 4 4 1 1 1 0 4 0 4\n",
      " 4 4 4 0 1 0 4 4 4 4 4 0 4 0 4 4 1 4 1 4 4 1 4 0 1 4 1 0 0 0 1 1 1 4 4 0 0\n",
      " 4 4 0 1 0 4 4 4 4 4 0 1 0 4 0 0 0 1 4 0 0 0 1 1 4 1 4 4 1 1 1 1 4 4 1 1 4\n",
      " 0 2 1 1 0 1 0 1 1 4 1 1 0 1 1 0 1 0 1 4 0 4 1 1 1 1 4 4 4 4 0 4 0 4 0 1 1\n",
      " 4 4 1 1 0 1 1 1 4 4 0 1 0 4 0 1 4 4 4]\n",
      "    Labels: [[0. 0. 1. 0. 0.]\n",
      " [0. 0. 0. 0. 1.]\n",
      " [0. 0. 0. 1. 0.]\n",
      " ...\n",
      " [0. 0. 1. 0. 0.]\n",
      " [0. 0. 1. 0. 0.]\n",
      " [1. 0. 0. 0. 0.]]\n",
      "Loss test: 1.6121267080307007\n",
      "Step: 0, Initial Loss: 1.6121267080307007\n",
      "Step: 1,         Loss: 1.610520839691162\n"
     ]
    }
   ],
   "source": [
    "batch_size = X_train.shape[0]#50\n",
    "\n",
    "features, labels = X_train, y_train\n",
    "features_val, labels_val = X_val, y_val\n",
    "\n",
    "features = tf.convert_to_tensor(features)\n",
    "labels = tf.convert_to_tensor(labels)\n",
    "features_val = tf.convert_to_tensor(features_val)\n",
    "labels_val = tf.convert_to_tensor(labels_val)\n",
    "\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((features, labels))\n",
    "train_dataset = train_dataset.shuffle(X_train.shape[0])\n",
    "train_dataset = train_dataset.batch(batch_size=batch_size)#X_train_04.shape[0]\n",
    "print(train_dataset)\n",
    "\n",
    "#for x, y in train_dataset:\n",
    "#    print(x.shape, y.shape)\n",
    "\n",
    "\n",
    "layer_dims = [[784,512],[512],[512,512],[512],[512,n_classes],[n_classes]]\n",
    "    \n",
    "model = tf.keras.Sequential([\n",
    "  tf.keras.layers.Dense(layer_dims[0][1], activation=tf.nn.relu, input_shape=(layer_dims[0][0],)), \n",
    "  tf.keras.layers.Dense(layer_dims[2][1], activation=tf.nn.relu),  \n",
    "  tf.keras.layers.Dense(layer_dims[4][1], activation=tf.nn.softmax)\n",
    "])\n",
    "\n",
    "# save weights to reinitialize later to same ones\n",
    "Wsave = model.get_weights()\n",
    "\n",
    "predictions = model(features)\n",
    "#predictions[:5]\n",
    "\n",
    "#tf.nn.softmax(predictions[:5])\n",
    "\n",
    "print(\"Prediction: {}\".format(tf.argmax(predictions, axis=1)))\n",
    "print(\"    Labels: {}\".format(labels))\n",
    "\n",
    "def loss(model, x, y):\n",
    "    y_ = model(x)\n",
    "    return tf.losses.softmax_cross_entropy(onehot_labels=y, logits=y_)\n",
    "\n",
    "l = loss(model, features, labels)\n",
    "print(\"Loss test: {}\".format(l))\n",
    "\n",
    "\n",
    "def grad(model, inputs, targets):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss_value = loss(model, inputs, targets)\n",
    "    return loss_value, tape.gradient(loss_value, model.trainable_variables)\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)\n",
    "\n",
    "global_step = tf.Variable(0)\n",
    "\n",
    "loss_value, grads = grad(model, features, labels)\n",
    "\n",
    "print(\"Step: {}, Initial Loss: {}\".format(global_step.numpy(),\n",
    "                                          loss_value.numpy()))\n",
    "\n",
    "optimizer.apply_gradients(zip(grads, model.trainable_variables), global_step)\n",
    "\n",
    "print(\"Step: {},         Loss: {}\".format(global_step.numpy(),\n",
    "                                          loss(model, features, labels).numpy()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(35000, 784) (35000,)\n",
      "Shape before one-hot encoding:  (35000,)\n",
      "Shape after one-hot encoding:  (35000, 5)\n"
     ]
    }
   ],
   "source": [
    "X_all = X.copy()\n",
    "y_all = y.copy()\n",
    "\n",
    "# building the input vector from the 28x28 pixels\n",
    "X_all = X_all.reshape(X_all.shape[0], 784)\n",
    "X_all = X_all.astype('float32')\n",
    "\n",
    "# normalizing the data to help with the training\n",
    "X_all /= 255\n",
    "\n",
    "print(X_all.shape, y_all.shape)\n",
    "\n",
    "# one-hot encoding using keras' numpy-related utilities\n",
    "print(\"Shape before one-hot encoding: \", y_all.shape)\n",
    "y_all = np_utils.to_categorical(y_all, n_classes)\n",
    "print(\"Shape after one-hot encoding: \", y_all.shape)\n",
    "\n",
    "features_all, labels_all = X_all, y_all\n",
    "features_all = tf.convert_to_tensor(features_all)\n",
    "labels_all = tf.convert_to_tensor(labels_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# indexes to update weights\n",
    "prev_idx = 0\n",
    "idx2 = []\n",
    "idx1 = []\n",
    "for i in range(len(layer_dims)):\n",
    "    idx1.append(prev_idx)\n",
    "    curr_idx = np.prod(layer_dims[i])\n",
    "    prev_idx = prev_idx + curr_idx\n",
    "    idx2.append(prev_idx)\n",
    "\n",
    "def dWdt(t, u, f_args):\n",
    "    \n",
    "    model = f_args[0]\n",
    "    x = f_args[1]\n",
    "    y = f_args[2]\n",
    "    layer_dims = f_args[3]\n",
    "    idx1 = f_args[4]\n",
    "    idx2 = f_args[5]\n",
    "    \n",
    "    [tf.assign(model.trainable_variables[i], \n",
    "               u[idx1[i]:idx2[i]].reshape(layer_dims[i])) for i in range(len(layer_dims))]\n",
    "    \n",
    "    #print(\"dWdt - iteration\",t, model.trainable_variables)\n",
    "    #print(\"dWdt - iteration\",t)\n",
    "    _, grads = grad(model, x, y)\n",
    "\n",
    "    # Minimize gradients, so add -1 *\n",
    "    new_grads = [-1 * p.numpy().flatten() for (i,p) in enumerate(grads)]\n",
    "    new_grads = np.concatenate(new_grads)\n",
    "    \n",
    "    return new_grads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from sys import stdout\n",
    "\n",
    "def train_tf(num_epochs, stiff_training, train_dataset, optimizer, max_epoch_no_impr = 50, epoch_to_vis=50):\n",
    "    \n",
    "    times = []\n",
    "    train_loss_results = []\n",
    "    train_accuracy_results = []\n",
    "    val_loss_results = []\n",
    "    val_accuracy_results = []\n",
    "    all_loss_results = []\n",
    "    all_accuracy_results = []\n",
    "    \n",
    "    best_validation_accuracy = 0.0\n",
    "    last_improvement = 0\n",
    "    require_improvement = max_epoch_no_impr\n",
    "    improved_str = ''\n",
    "    impr_perc = 0\n",
    "    best_weights = model.get_weights()\n",
    "    \n",
    "    start_time = time.time()\n",
    "    \n",
    "    # numpy initial weights\n",
    "    w0 = [p.numpy().flatten() for (i, p) in enumerate(model.trainable_variables)]\n",
    "    w0 = np.concatenate(w0)\n",
    "\n",
    "    ##### ODE solver - initial settings\n",
    "    t0 = 0.0\n",
    "    tfin = 0.125\n",
    "    solver = ode(dWdt)\n",
    "    #solver.set_integrator(\"vode\", atol=1e-8, rtol=1e-6, method=\"bdf\") #adam, non-stiff \n",
    "    solver.set_integrator(\"zvode\", atol=1e-8, rtol=1e-6) \n",
    "    #####\n",
    "    for epoch in range(num_epochs):\n",
    "        epoch_loss_avg = tfe.metrics.Mean()\n",
    "        epoch_accuracy = tfe.metrics.Accuracy()\n",
    "        epoch_loss_avg_val = tfe.metrics.Mean()\n",
    "        epoch_accuracy_val = tfe.metrics.Accuracy()\n",
    "        epoch_loss_avg_all = tfe.metrics.Mean()\n",
    "        epoch_accuracy_all = tfe.metrics.Accuracy()\n",
    "\n",
    "        # Training loop - using mini-batches\n",
    "        btch_n = 1\n",
    "        for x, y in train_dataset:\n",
    "\n",
    "            loss_value, grads = grad(model, x, y)\n",
    "            \n",
    "            if stiff_training:\n",
    "\n",
    "                solver.set_initial_value(w0, t0)\n",
    "                solver.set_f_params((model, x, y, layer_dims, idx1, idx2))\n",
    "                solver.integrate(tfin, step=False)\n",
    "                w0 = solver.y\n",
    "                #update weights\n",
    "                [tf.assign(model.trainable_variables[i], \n",
    "                           w0[idx1[i]:idx2[i]].reshape(layer_dims[i])) for i in range(len(layer_dims))]\n",
    "\n",
    "                loss_value, grads = grad(model, x, y)           \n",
    "                \n",
    "            else:            \n",
    "                #update weights\n",
    "                optimizer.apply_gradients(zip(grads, model.trainable_variables),\n",
    "                                          global_step)\n",
    "\n",
    "\n",
    "            # Track progress\n",
    "            epoch_loss_avg(loss_value)  # add current batch loss\n",
    "            epoch_loss_avg_val(loss(model, features_val, labels_val))  # add current validation\n",
    "            epoch_loss_avg_all(loss(model, features_all, labels_all))  \n",
    "            # compare predicted label to actual label\n",
    "            #epoch_accuracy(tf.argmax(model(x), axis=1, output_type=tf.int32), y)\n",
    "            epoch_accuracy(tf.argmax(model(x), axis=1, output_type=tf.int32), \n",
    "                           tf.argmax(y, axis=1, output_type=tf.int32)) \n",
    "            epoch_accuracy_val(tf.argmax(model(features_val), axis=1, output_type=tf.int32), \n",
    "                           tf.argmax(labels_val, axis=1, output_type=tf.int32)) \n",
    "            epoch_accuracy_all(tf.argmax(model(features_all), axis=1, output_type=tf.int32), \n",
    "                           tf.argmax(labels_all, axis=1, output_type=tf.int32)) \n",
    "\n",
    "            btch_n = btch_n + 1\n",
    "            \n",
    "        #print(\"End of batch -- time (s): {:03f}, Epoch: {:03d}, batch_no: {:03d}, Loss: {:.3f}, Accuracy: {:.3%}, Val Loss: {:.3f}, Val Accuracy: {:.3%}, Full Loss: {:.3f}, Full Accuracy: {:.3%}\".format(time.time() - start_time,\n",
    "        #                                                                                j, btch_n,\n",
    "        #                                                                                epoch_loss_avg.result(),\n",
    "        #                                                                                epoch_accuracy.result(),\n",
    "        #                                                                                epoch_loss_avg_val.result(),\n",
    "        #                                                                                epoch_accuracy_val.result(),\n",
    "        #                                                                                epoch_loss_avg_all.result(),\n",
    "        #                                                                                epoch_accuracy_all.result()))\n",
    "        \n",
    "        # end epoch\n",
    "        train_loss_results.append(epoch_loss_avg.result())\n",
    "        train_accuracy_results.append(epoch_accuracy.result())\n",
    "        val_loss_results.append(epoch_loss_avg_val.result())\n",
    "        val_accuracy_results.append(epoch_accuracy_val.result())\n",
    "        all_loss_results.append(epoch_loss_avg_all.result())\n",
    "        all_accuracy_results.append(epoch_accuracy_all.result())\n",
    "        times.append(time.time() - start_time)\n",
    "        \n",
    "        if epoch_accuracy_val.result() - best_validation_accuracy >0.005:\n",
    "            # Update the best-known validation accuracy.\n",
    "            impr_perc = epoch_accuracy_val.result() - best_validation_accuracy\n",
    "            best_validation_accuracy = epoch_accuracy_val.result()\n",
    "            best_weights = model.get_weights()\n",
    "            \n",
    "            # Set the iteration for the last improvement to current.\n",
    "            last_improvement = epoch\n",
    "\n",
    "            # A string to be printed below, shows improvement found.\n",
    "            improved_str = 'BEST (Val): ' + '{:.3%}'.format(epoch_accuracy_val.result())\n",
    "        else:\n",
    "            impr_perc = epoch_accuracy_val.result() - best_validation_accuracy\n",
    "\n",
    "        if epoch % epoch_to_vis == 0:\n",
    "            stdout.write(\"time (s): {:03f}, Epoch: {:03d}, Loss: {:.3f}, Accuracy: {:.3%}, Val Loss: {:.3f}, Val Accuracy: {:.3%}, Full Loss: {:.3f}, Full Accuracy: {:.3%}, {}, last best: {}, impr %: {:.3%}\\r\".format(time.time() - start_time,\n",
    "                                                                                    epoch,\n",
    "                                                                                    epoch_loss_avg.result(),\n",
    "                                                                                    epoch_accuracy.result(),\n",
    "                                                                                    epoch_loss_avg_val.result(),\n",
    "                                                                                    epoch_accuracy_val.result(),\n",
    "                                                                                    epoch_loss_avg_all.result(),\n",
    "                                                                                    epoch_accuracy_all.result(),\n",
    "                                                                                    improved_str,\n",
    "                                                                                    -epoch + last_improvement,\n",
    "                                                                                    impr_perc))\n",
    "            stdout.flush()\n",
    "\n",
    "        # If no improvement found in the required number of iterations.\n",
    "        if epoch - last_improvement > require_improvement:\n",
    "            stdout.write(\"\\n\") # move the cursor to the next line\n",
    "            print(\"time (s): {:03f}, Epoch: {:03d}, Loss: {:.3f}, Accuracy: {:.3%}, Val Loss: {:.3f}, Val Accuracy: {:.3%}, Full Loss: {:.3f}, Full Accuracy: {:.3%}, {}, last best: {}, impr %: {:.3%}\".format(time.time() - start_time,\n",
    "                                                                                    epoch,\n",
    "                                                                                    epoch_loss_avg.result(),\n",
    "                                                                                    epoch_accuracy.result(),\n",
    "                                                                                    epoch_loss_avg_val.result(),\n",
    "                                                                                    epoch_accuracy_val.result(),\n",
    "                                                                                    epoch_loss_avg_all.result(),\n",
    "                                                                                    epoch_accuracy_all.result(),\n",
    "                                                                                    improved_str,\n",
    "                                                                                    -epoch + last_improvement,\n",
    "                                                                                    impr_perc))\n",
    "            print(\"No improvement found in a while, stopping optimization.\")\n",
    "            # Break out from the for-loop.\n",
    "            break\n",
    "\n",
    "        t0 = solver.t\n",
    "        tfin = solver.t * 2\n",
    "\n",
    "            \n",
    "    stdout.write(\"\\n\") # move the cursor to the next line\n",
    "            \n",
    "    return train_loss_results, train_accuracy_results, val_loss_results, val_accuracy_results, all_loss_results, all_accuracy_results, best_weights, times\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tensorflow.python.training.gradient_descent.GradientDescentOptimizer object at 0x1c504a5128>\n",
      "time (s): 88.328985, Epoch: 200, Loss: 1.048, Accuracy: 88.600%, Val Loss: 1.090, Val Accuracy: 83.400%, Full Loss: 1.089, Full Accuracy: 83.317%, BEST (Val): 85.400%, last best: -11, impr %: -2.000%\n",
      "time (s): 105.511589, Epoch: 240, Loss: 1.035, Accuracy: 90.000%, Val Loss: 1.085, Val Accuracy: 83.000%, Full Loss: 1.084, Full Accuracy: 83.437%, BEST (Val): 85.400%, last best: -51, impr %: -2.400%\n",
      "No improvement found in a while, stopping optimization.\n",
      "\n",
      "<tensorflow.python.training.momentum.MomentumOptimizer object at 0x1c504a5400>\n",
      "time (s): 44.019544, Epoch: 100, Loss: 0.995, Accuracy: 93.000%, Val Loss: 1.083, Val Accuracy: 82.000%, Full Loss: 1.077, Full Accuracy: 82.494%, BEST (Val): 85.200%, last best: -39, impr %: -3.200%\n",
      "time (s): 49.564251, Epoch: 112, Loss: 0.970, Accuracy: 94.000%, Val Loss: 1.071, Val Accuracy: 82.600%, Full Loss: 1.067, Full Accuracy: 83.674%, BEST (Val): 85.200%, last best: -51, impr %: -2.600%\n",
      "No improvement found in a while, stopping optimization.\n",
      "\n",
      "<tensorflow.python.training.adadelta.AdadeltaOptimizer object at 0x1c504a55c0>\n",
      "time (s): 91.573524, Epoch: 200, Loss: 1.073, Accuracy: 89.000%, Val Loss: 1.097, Val Accuracy: 84.400%, Full Loss: 1.097, Full Accuracy: 84.106%, BEST (Val): 85.000%, last best: -9, impr %: -0.600%%\n",
      "time (s): 110.399443, Epoch: 242, Loss: 1.056, Accuracy: 89.000%, Val Loss: 1.087, Val Accuracy: 85.000%, Full Loss: 1.087, Full Accuracy: 84.529%, BEST (Val): 85.000%, last best: -51, impr %: 0.000%\n",
      "No improvement found in a while, stopping optimization.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "### Full dataset\n",
    "\n",
    "num_epochs = 5001\n",
    "stiff_training = False\n",
    "\n",
    "optimizers = [tf.train.GradientDescentOptimizer(learning_rate=0.1),\n",
    "              tf.train.MomentumOptimizer(use_nesterov = True, learning_rate = 0.1, momentum=0.9),\n",
    "              tf.train.AdadeltaOptimizer(learning_rate=0.1)\n",
    "             ]\n",
    "\n",
    "train_loss = []\n",
    "train_accuracy = []\n",
    "val_loss = []\n",
    "val_accuracy = []\n",
    "all_loss = []\n",
    "all_accuracy = []\n",
    "weights_save = []\n",
    "train_times = []\n",
    "for optimizer in optimizers:\n",
    "    print(optimizer)\n",
    "    model.set_weights(Wsave) # reset weights as for the previous training\n",
    "    train_loss_results, train_accuracy_results, val_loss_results, val_accuracy_results, all_loss_results, all_accuracy_results, best_weights, times = train_tf(num_epochs, stiff_training, train_dataset, optimizer, epoch_to_vis=50)\n",
    "    train_loss.append(train_loss_results)    \n",
    "    train_accuracy.append(train_accuracy_results)\n",
    "    val_loss.append(val_loss_results)\n",
    "    val_accuracy.append(val_accuracy_results)\n",
    "    all_loss.append(all_loss_results)\n",
    "    all_accuracy.append(all_accuracy_results)\n",
    "    weights_save.append(best_weights)\n",
    "    train_times.append(times)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.set_weights(Wsave) # reset weights as for the previous training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time (s): 871.094145, Epoch: 030, Loss: 1.018, Accuracy: 92.000%, Val Loss: 1.073, Val Accuracy: 84.600%, Full Loss: 1.071, Full Accuracy: 84.789%, BEST (Val): 85.200%, last best: -11, impr %: -0.600%\n",
      "time (s): 871.095756, Epoch: 030, Loss: 1.018, Accuracy: 92.000%, Val Loss: 1.073, Val Accuracy: 84.600%, Full Loss: 1.071, Full Accuracy: 84.789%, BEST (Val): 85.200%, last best: -11, impr %: -0.600%\n",
      "No improvement found in a while, stopping optimization.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 501\n",
    "stiff_training = True\n",
    "\n",
    "train_loss_results_ode, train_accuracy_results_ode, val_loss_results_ode, val_accuracy_results_ode, all_loss_results_ode, all_accuracy_results_ode, best_weights_ode, times_ode = train_tf(num_epochs, stiff_training, train_dataset, optimizer=None, max_epoch_no_impr=10, epoch_to_vis=2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzde3iU1bX48e+aSUIQMEFEQe4CokAgAQTjBYIoIlhFsYr2FO+2FVTUc1R6fnKotMVab6WlFxRvrRUUC0VLKxWJoMYKGEDFIlQCREAwkHBnkpn1+2MuTJKZZCaZSTKZ9XmePMm81z0zsNf77r3ftUVVMcYYk7wcjV0AY4wxjcsCgTHGJDkLBMYYk+QsEBhjTJKzQGCMMUnOAoExxiS5lMYuQLROPfVU7d69e2MXwxhjEsratWu/VdX2odYlXCDo3r07a9asaexiGGNMQhGRbeHWWdOQMcYkOQsExhiT5CwQGGNMkrNAYIwxSc4CgTHGJLmEGzVkjGn6CsrKyC8tJS8zk9yMjMDrdqmplJSXB5abpsECgTEJqqAA8vOhXTsoKQn/Oy8PcnNjeN6gSh6oVMG3S02l8OBBXti9mwpV0hwO7u7UiaeLi6lQRfE2Q6SIcGuHDuS0aRM2MFQNJvUppz8Yvbx7NwCTOnRosoGoYEcBj3/wOIW7CznuPo7L7UJVyTo9i8dGPUZulxh+mT6SaPMRDBkyRO05ApMI/BW1vyIOfg2VK/HgZbWtb9cOCgvhhRegvBw8HhAB1eq/HQ5o0QKWL69bMKha6b+8e3egkneKIEC5Kh5AAA36DSfanj01nCM4MPgr6IKyMkatX89xjwcBslq14rgqfU46iQe7dKm1Ep+7cydTNm+mQhWHb/8Nhw8HyuEALszIoO9JJzVaUJi7di7PfPQM+4/tJz0lncz0THYf3M3uw7sD24xbM44JH02g9bHWlKeU858O/+HiWRdzwfgLoj6fiKxV1SEh11kgMCY6oSr0qpV9u3YwdSq4XJCWBs88c+K10+mtqP2VuMMBKSneZRUVNa+vWvFHwumEmTNh2rTQV/Phmm/apaYydcsWXB5PoNJ3+a7qwVvhw4lKvyoBnCK4g/bxLw+3T4oI93fuzLpDh/jn/v1ht+veogWZKSkcV6WFSOD3/ooKjns8fFNeHnbfqlqIsCI7O2bBYO7aucz7ZB4uj4vjFcdpkdIi8Hv/0f2ICBXuCooPFtN3R1+u/+B6eu3qRao7tdJxWh9tTbo7vdrxNUUZvHIwGbnRlbemQGBNQyZpVb1ij2QZwKhRlSv0iorqlb2It8L2eLyv33jD+9vt9i6DExW5x+Ot4P3Lalsf/Nuv8p2ABmpbh0NISVO299nN3J1arWL3N98806sXU7ds4bjHgwfvFbNDBI/vat/jO2FwEEitEhyC7wgcwAUZGZySksLf9+2jXBWHr5I/UFHBC7t3V7uTwFeex3fsqDFYABQdPw7Hj9ewReRcquSXlkYcCAp2FPDy+pfZfch75b7v6D72HtlLi5QW1a7o/fxX9pmHM0kvT8fpceLwOHDirPV8Egi5vtcVQml+adSBoCYWCEyzEqoiD7W8oOBEhZ6W5m06gdqX3XRT6Aq9amXvcJwIFGlpMGECrFoVuzsC/7Jbb4WcnBN9AoVfH+O5w9twl6XizCjnitSOLO3yH549pQzZHLpid3k8vLF3Ly5fEABfU46/+cf3W1XxxSMcwNTOnfnyyBHeLCkJBI+sVq341NcEs7KsLNDsc2W7dgAcqKggp02bwPeS06YNhQcP8uyuXbiDvsd4tVOEuosRoF3qiavxcBV9+1btQWHV9lVolRKOWzOOsYVjaXW0FacePJUUz4mq1eFxkKLhq9qqFX1NFEVEyMzLjHifSFggMAkp3JV71Yo8XKWfn3+i0na5vK+h9mXgPUaoO4Lgyt5/hxDcWZuVFbs+guDj+pt0sjIzKSktRbfuQvFWdkfaHsO9vww34KhSsQffEUxo355VZWWV7gha+O4U/M1E83btYvXBg4FjB3cAA7iBdYcPV/qePL5zvFlScqKi37UrcPxJHTpw5xlnkNOmTaBNP9TdRVarVuyvqGBblHcBTuA7viDk9/mBEja7TlTkHuBHmzbySsFMOLCR97e/jydEr4YUCre/czsP7nwQhzood5TjxInD7QjZhFNt/ygqfL+qAQeg5PslMb0bAAsEJgGFq/BDVe7hluflnajQ09JOVLS1LZs0yfsTro8guLKv2jmbm1t5WajO20jW+yt+MjMpKINR69fj8ngCTTxpDkfgtb+CD17vH6UDlfsIslq1CjnEM7jj1j/qxxGi7T+U4G2DeYDjHg8zioqY0b07d55xRqXzv7F3L+/s3x+ojq877TSmdetGQVkZj2/fzqajRyv1DQT3EZRVVFDq9oYdt3pYt30FXY99wapTJqDBTTFyomL2qIOVZQdgx0oAbv/n7VxeeDmi3m3Sy9NJr6hc2adT+XWkFX3tn1plLoeLipQKyk4q488X/pnyS8qZwISojlEbCwSmSQrXxAPhK/xwlXuo5bm5J+4Mgs8R6bJwFXbVyj7W/JWyv2K/6fTTcXk8uPE28ZSUl7N84MBqFbx/2GRWq1aVOob9AWHWtm3kZWYyrVu3SueatW0b248dCzQbOYBL2rZlQvv2lfobxp5yCn/fty8wmih4aGi71FTu3rwZV1AwELzB4J39+8kvLQ2MGPKfP6tVq0oBzF/O3IwMFmVlVf9cgptzTu7Hmy1H4sYBCEUtz6FI23hLH1T5o8q4t2DCG9D6IFB+Leh4Tjp2EidVnBTy84+kso+konfjRkXxiAePw8Oh9EPsa72PNkfbBDqND7U8xBvD3uBvQ/5Wad8/9P1DrcePlo0aMk1OuCv+SNZH2kfQFIUaN1912axt23hk61bceJs97ujYkZe++SZQYS4fOLDavsGjf4I7hsN1GocbMeTfxn+OcA+NhXsmwB+Mctq0Yd6Ozaw+6kF9FasA6WHKX/V4wZV+h9YdODn9ZJ788EncGtTL0HsqdPwOiCPQs16p0gdaHFPaHAlfsYer9COp6CukAo/jRPNShbOCktYlHGp5iKU5S6tV7pHontmdaRdO487Bd0a9L9jwUdME1VQxz5oFjzziveIPHvoY6f6JqOqV/vKBAwEiXla1SSd4Owka/eMERrVty/L9+3FTufPUgXe4p0e12j53dOxI1/T0Oj3c5a+4AXI65lC4q5B5//mI8v6zQFK9lbV4ewWGtnTwzNknhnIW7Cggvyifdie1o3BXIbsP7eZvm/9Guae8hjPCuM9vZ8K/JtLCJRxsJZy2R8g86H+nlUVb4SuKy+nC5XThxIkbNzig9KTSWiv6Dq070KF1h8Aw0sz0TPYf3c9xd+i+j/SUdLI7ZPPg+Q/W+0EyGz5qmpTarvjDNfEEi3cTTEPLLy2t1MSTX1oKvr+Dl03r1i3Q9NMuNTVwxZyXmRm2ySi4k7hqv0Hw1b742vI9Ifap7aGrUBW2X9iKe8MDcNpo6Hg5qAPEycdHKhj+yRqucBSxr6KCDz79LZ6yT8NWyv5x+F2+7YIrxUWbo21o6WpJ5tETo2o6Av4AEKrSr+kK/0jKEVypLtLcaRxJO8IXnb9gwQUL2NhlY9h9OrTuQHbr7ErPEPQ5tU9MKvN4sUBgGly4Nn6/cO33zVleZmalTl7/VX3wsnapqYG2/JoqfpdvXGvwvsGdxMEdw1XTRFRtQqrUaVylScZ/db9x70Y+2PFB5aaZSBzY6P3Zswy63QRtB4M4qfC4WezuBg6BrF/Ahgfo+zmMXj+atofa0uZoGzIPZ+LwOOiyr0vYK/poKv2DaQc5nnbiqjxc+7xfdodsTk47udKw0mMVx7ht0G11brppTNY0ZGIqkiab2u4IkkGoJGxAjTl8givpm04/PTD2PlxfQfDxImnOKdhRwOOf/o0v3S05y3mUy0/vSuGuQoDQ7fD1JAjiez5BTz6HvilPM/qfqbTd513f5oCQecCDy11Cr+JTcIRJlhxt005wpV9bhS8IF3W7iFPSTwG8V/uTBk5qslf2NWm0piERGQP8Cu+/1edU9bEq67sCLwGZvm0eVtWl8SyTiZ9IK/jGuOKvbwKzWB43eDhm8Jj95QMHBoZIVu0HqNp0BJWv+Cf5Rt1U6rzdUQDb8/n023a87KvQczrmUHKkJNCE419WuKuQeYXzAk04G4HFMfqMnOLEox7O2XFO4Ko+mAMHZ8qZnP5Fiq+q9z9F4H+S4FTf0ujb8ktalwRG4xxPO16t0nfg4MJuFzI+fXy1/RO50o9W3AKBiDiBOcClQDGwWkSWqGpw49r/A15T1d+JSF9gKdA9XmUy8VVbk0+w+rbxR1MBh6pY6xMMwo3GifS4/ko9+Clefx9AbkZGyP6C4KajFAG+Wcbd6RmsO6pkpwv5Gz6g3Unt4EhJoOJ/Yd0LlLvLqz0cJUi1yjPUsmj03dGX0etHc8oh75Vzu5btSHOmQXvoM7wP7kVu9OPaj3+iso+u0v+29beBCh+BLR22sOCCBWzquokHzn+AA8cOsPvQblJJZTzeSj+ZKvraxPOOYCiwRVW/AhCR+cBVeC82/BQ42fd3BrAzjuUxcRZJJ28sRFuxh6pY6xoIgs8dPLImmuP6K/XgO4LgfoG8zExSxJsGwoGyfdsScAzhmY5O5n21jsJ/v8Dc0g148CAIy9BARV71dyihlkcSBATBIQ6ud13PsB3DOK3zaRwqPET61nTO2HRG4OGrqlfu7r+7Qy4Px38fEKpcHjxs6bCFtIo0XCku3E53YJROqiOVcb3HBbYd3no4zw18zir6CMQzEHQCdgS9LgaGVdlmBrBMRO4GWgGXxLE8Js6iafKp7Yq+pvU1Veyh9gvXEVsXweeuOrKm6nHDvYfcjIxAc0/pwe2sK/mKCR17wYGNzNrgHXmj638PrftSXlbI3IP/5vmVKQjizU0fVDn6/w73O5RQQcIhDlIcKYztNRagUmcwQO7eXFL/mkrH3R1hDd7bmBh0L4Yqp4hwOPswW/Zt4eTDJ1PaqpSDLQ+yv/V+lg1cVmnEjj84je8zvkmPymnq4hkIQoX/qt/6DcCLqvqkiOQCfxSR/qpa6V5WRO4E7gTo2rVrXAprqqvLWP1Imnxqu6KvbX24ij3cfsEVb337CKqeu+rImkjfAwc2sv3Tl3lh3QtUeCrIdzgRhApPhe9Ow4Pu91bCHqDc7W27D9ekU/W3v2K/NfvWShV6uD6CkiMl5HXPq1SRlhWUMeJvI3DtdlHytxKoeeh+xFpltyK9uzc9w76j+7y5fdSDopS2LuWf2f9kY5eNeLR6vh+nOBl/1vhAoApVbhO9eAaCYqBL0OvOVG/6uQ0YA6CqBSKSjrdnaE/wRqo6F5gL3lFD8SqwOSGeI3tqa6qpbX24ir2m/fwBob4iDSohnws4sDEw1n7qP6ZyrOJYoGL3uL2VnqI41IHT4QT1NoX4K3V/oHA6nIEK3l+ph/pdlwqyrKCM3S/vrlvl74CMCzNIOeVEtZLWIY02OW0o+XsJrp0uOt7WkTPuPKPSbmk70piRP4N/fvXPE4GuapptBKfDyZyxcxJyeGZTF89AsBroLSI9gK+BicCNVbbZDowCXhSRc4B0YG8cy2QiFE3Hb7Rqa6qJpCknVMUeyyagmkQSVKqWpZ2rmFGvXYrL7TpxxR/0kFOqMzVQ0ac503hmzDPVKnWA/KL8uF0B75y7ky/v+hIiGSHqhC4PdKHiQAWu3S7SOqTRYVKHsFkxq1b+wXK75DIjbwbvFr1Lhaei0roURwr3595PZotMu/KPo7gFAlWtEJEpwNt4h4Y+r6qfi8ijwBpVXQI8ADwrIvfhvQa4WRPtwYZmKp4dv7VdVde1KSeWTUB1Fdwv8ExHJ2/s3MqEjr0o2b0Sl9uFW92BK37/Ve6t2bcyaeAkoPaKPh4Vof8uYNezu8IHgVRoN86bzrm2Sr8ucrvkMmfsHKYsnYJb3YFmLRvV0zDsgTITVnPL51NVrJ8tCO4XSBHQ9ffjLt0QuMK/+4PfU96mH6kHP+fXF/ywSbRvlxWUsX7UejzHPNV78HyVfzwq/nD8qSoa+3NpjizXkKmT5pbPJ1isny0Ab7+Af0iox+OBNv3Q/YW43C4KDx9HBj4Fvpw+Wb1yGuWOJVhZQRlFM4rwHK8SBJxw6ndOpcuDXRqk8g+W2yXXAkAjsEBgklIsny3wa+cqxuM+BpKCagWpBz7DI07vg1WZ2VTsc6MIFUpMzlcfO+fuZPOUzWiFBlKPSorQ4dYODXb1b5oOCwQmKcWjY7lk90ocG/6MJ2MAjrIN3Nb7QrpmXOft6D25Ly+Vro97R3YkygrK2DzZFwQABNpe0pbuM7pbAEhSFghMUgrXsVyffoO87nm0WDkT18EvSHOmMWng3ErNHI3dkQ0nmoPUHTRbWIpYEEhy1llsjE8s+g2acmdnoGP4uMf7lJqAOIXec3rXOLzTNA/WWWxMBOrSb1C14m+qnZ2VOoZ9CY6sOcj4WSAwMRWvdM8NIdp+g4IdBYx6eRQut4s0ZxrLJy1vskGg0p2AAxwtHBYETIAFAhMz8RiS2ZCifSAtvyg/8JCYy+0ivyi/SQaC3S/vPvGcgN0JmBAsEJiYiceQzIYWTU6ivO55pDnTAncE/jQQTUlZQRm7n98deE5AUq1j2FRngcDETEPl+mkqcrvksnzS8ibbOQxQml96YoSQQIdb7BkBU50FAhMzTSHXT0Nrqp3Dfpl5mTjSHHhcHhxpDjpM6tDYRTJNkAUCE1OxSvfclDXlIaLBygrKKM0vpdczvSgvKSczL9PuBkxIFgiMqUXwSCgObEyskUK+O4GBywdaEDBhWSAwpgZVR0LdxPqEGClUml+Kx+UBN3hcHkrzSy0QmLAcjV0AY5qyqiOhyMwmzZmG05dMrqmOFDq2/RiSIuAER5qDzLzm3XFv6qfWOwIROQ24ADgDOAp8hndimeoTihrTzFQdCTWp2wAmNeGRQsFNQuIUOt7R0bKJmlqFDQQiMhJ4GDgFKMQ7j3A6MB7oKSILgSdV9UBDFNSYxhByJFRG0x0pFNwkpCjpXdMtCJha1XRHMBa4Q1W3V10hIinAFcClwBtxKpsxTUIijYSqOlzUmoRMJMIGAlX9HwAR6aGqW6us7qKqi+NaMmNMVGy4qKmrSEYNvQEMqrJsITA49sUxpulJhOcGbLioqY+a+gjOBvoBGSJyTdCqk/H2FRjT7CVKhlEbLmrqo6bho33w9gNkAt8J+hkE3BHJwUVkjIhsEpEtIvJwiPVPi8g638+XIlIa/VswJn5CZRhtivx9AzZc1NRFTX0EfwX+KiK5qloQ7YFFxAnMwduhXAysFpElqrox6Bz3BW1/N5AT7XmMiadEyTBqfQOmPiLpIygRkeXA6araX0QGAFeq6k9r2W8osEVVvwIQkfnAVcDGMNvfAPxfhOU2pkE09Qyj1jdgYiGSJ4ufBaYB5QCqugGYGMF+nYAdQa+LfcuqEZFuQA/g3TDr7xSRNSKyZu/evRGc2pjYye2Sy7SLpjW5IACh+waMiVYkgeAkVf24yrKKCPaTEMs0zLYTgYWq6g61UlXnquoQVR3Svn37CE5tTHKwvgETC5E0DX0rIj3xVeIici2wK4L9ioEuQa87AzvDbDsRmBzBMY0xPtY3YGIlkkAwGZgLnC0iXwNbgf+KYL/VQG8R6QF8jbeyv7HqRiLSB2gLRN0hbeIrkSeir6tEeGYArG/AxFatgcDX2XuJiLQCHKp6MJIDq2qFiEwB3gacwPOq+rmIPIo3ad0S36Y3APNVNVyzkWkEiT4RfV0kyjMDYM8NmNiqtY9ARO4VkZOBI8DTIvKJiIyO5OCqulRVz1LVnqr6M9+y6UFBAFWdoarVnjEwjSvURPTNXaI8M2Bppk2sRdJZfKsvw+ho4DTgFuCxuJbKNDp/+mUnJMVE9HDimYGmPtfA+lHr2fXsLlDoeEdHaxYy9RZJH4F/9M9Y4AVVXS8ioUYEmWYkWSeib8rPDIClmTbxEUkgWCsiy/CO858mIm0Am5QmAdS3szeR0i/XRaiO4dwuTXeuAbA00yY+IgkEtwHZwFeqekRE2uFtHjJNWDJ29kYjkTqG/Wy4qImXSEYNeURkK3CWiFjW0QQRqrPXAsEJoTqGm3IgsOGiJp4iGTV0O7AS7zDQn/h+z4hvsUx9JWNnbzQSoWM4mKWSMPEUSdPQvcC5wEeqOtI3T8FP4lssU1/J2NkbjUToGIYTzUGp7VKtb8DETSSB4JiqHhMRRKSFqv7b9zSwaeKae2dvNBKxY7hqc5D1DZh4iSQQFItIJrAY+KeI7Cd8ziBTB8mYyqEhJWrHcNGMIjzHPeDxNgeVl5TTbVq3xi6aaYYi6Sy+2vfnDBFZAWQAf49rqZJIPEf3FBRAfj7k5UFu06734iphO4Z9QQCHPT1s4qvWQCAif1TV7wOo6nv+ZcD341y2pBCv0T0FBTBqFLhckJYGy5cnRzAI1QSUCLOMwYn+gGPbj3k7hn1BoO0lbek+o7s1B5m4iaRpqF/wC98UlIPjU5zk4x/d478jiNXonvx8bxBwu72/8/ObVyAIVeGHawJqyh3DwZ3BW6ZuwePyIE5BUgRFcaQ5LAiYuAsbCERkGvBjoKWIHPAvBlx401KbGIjX6J68PO+dgP+OIC8vJoeNmdrSPde0PlyFX1MTUFPrGC4rKGP3y7vZ/cJutEIREdSj4PGmjuh4R0fSu6Zbx7BpEDVNXj8LmCUis1R1WgOWKenEY3RPbq63Oaih+wgiyedfW+dtbevDVfixaALyX6EHV8BVlwW/BqJeF7j6P+YJzNmnDkWcgor3LqDDpA4WAEyDiaSzeJqItAV6A+lBy1fGs2Cm/nJzG7Y5KNLRObV13ta2PlyFH20TUKgKvurTu0C1IZzBTTgIaIVGt85/9e+fgUPA0cKGh5rGE0ln8e14HyrrDKwDzsM7m9jF8S2aSTSRjs6p7cq9tvU1VfiRNgGFqvTDPb0bvGzvG3tPZP/0+C/no1wXdPUvTqHDrR3sDsA0KnuyuAloLsM8I22aqe3KPZIr+/q2+Yeq9MNl9gxe1n5Ce8pWlYW86o90nT0cZpoaqW2GSBFZrarnisg6YJiqHheRdaqa3TBFrGzIkCG6Zs2axjh1XDS3YZ5Ndc7fSJqBqrbxx6uPwCp/0xhEZK2qDgm5LoJAsAhv2umpeJuD9gOpqjo21gWNRHMLBLNmwSOPeId5Op0wcyZMayJd8021Uo9WNJW+Mc1VTYGgrk8W/yOG5UtqTXWYZyKmZQgn3ETv/h9jkl0kncVZwNm+l1/4ny42sdFYwzxrk2hpGWpis3oZU7OaHijLAP4KdAXW432YLEtEtgNX+Sa0r5GIjAF+BTiB51S12qT3InId3vkNFFivqjfW4X0ktIYe5hmJREnLUFWo5p6M3IzAqCBrBjKmurB9BCIyG+9TxA+qqse3zAE8BrRU1btrPLA3FcWXwKVAMbAauEFVNwZt0xt4DbhYVfeLyGmquqem4za3PoKmLNH6CGwWL2PCq2sfwSXAAH8QgMC0lT8GPo3gvEOBLar6la8Q84GrgI1B29wBzFHV/b7j1xgETMNqamkZahOuL8AYU7Oapqp0qWpF1YW+ZccjOHYnYEfQ62LfsmBn4Z0L+QMR+cjXlFSNiNwpImtEZM3evXsjOLVJRv6+AJyWttmYaNR0R5AuIjl4+waCCdAigmNX3Q9OPFQffP7eQB7eJ5dXiUh/Va00IauqzsWX6G7IkCE1j3c1Scv6Aoypm5oCwS7gqTDrdkdw7GKgS9DrzlSf2awY7xPL5cBWEdmENzCsjuD4xlRjQ0KNiV5N2UdH1vPYq4HeItID+BqYCFQdEbQYuAF4UUROxdtU9FU9z2uMMSYKNfUR1IuvL2EK8DbwBfCaqn4uIo+KyJW+zd4GSkRkI7AC+B9VLYlXmUzzUVZQxrZZ2ygrKGvsohiT8GpNMdHU2PBRY8NEjYleTcNH43ZHYEy8hEsXbYypm1oDgXj9l4hM973uKiJD4180Y0KzYaLGxFYk8xH8FvDgzTz6KHAQeAPvHAXGNDgbJmpMbEUSCIap6iARKQTwpYJIi3O5jKmRDRM1JnYi6SMo9+UNUgARaY/3DsEYY0wzEEkgmA0sAk4TkZ8B7wM/j2upTEwU7Chg1qpZFOwoaOyiGGOasEgmpnlFRNYCo/CmjRivql/EvWSmXprTxDLGmPgKe0cgIqf4f4A9wKvAn4FvfMtMExZqYhljjAmlpjuCtXj7BcIljzszLiUyMZGoE8v42XzCxjScmnIN9WjIgpjYyu2Sy/JJyxNqYhk/e3LYmIYVyfBRfLmBhvte5qvqW/ErkomVRJtYxs8mmDGmYUXyZPFjwL14ZxbbCNwrIrPiXTCTvOzJYWMaViR3BGOB7KB5i18CCoFp8SyYSV725LAxDSuipiEgE9jn+9v+V5q4syeHjWk4kQSCWUChiKzAO4JoOHY3YIwxzUYkD5S9KiL5eJPMCfCQqkYyVaUxxpgEEOl8BA7gW2A/cJaIDK9le2OMMQmi1jsCEfkFcD3wOSeSzSmwMo7lMsYY00Ai6SMYD/RR1ePxLowxxpiGF0nT0FdAarwLYowxpnGEvSMQkV/jbQI6AqwTkeVA4K5AVe+Jf/GMMcbEW01NQ2t8v9cCS+pycBEZA/wKcALPqepjVdbfDPwS+Nq36Deq+lxdzmUSiyWVM6bpqCkQlAEfquqeuhzYN6vZHOBSoBhYLSJLVHVjlU0XqOqUupzDJCZLKmdM01JTH8F/4X2QbLOIvCgid4pIvyiOPRTYoqpfqaoLmA9cVZ/C1kdBWRmztm2joKyssYpgfEIllTPGNJ6a0lBfCyAi3YHzfT8/EJGuwGpVHVvLsTsBO4JeFwPDQmw3wfdcwpfAfaq6o+oGInIncCdA165dazltdQVlZYxavx6Xx0Oaw8HygQPJzbAr0MbiTyrnvyOwpHLGNK5aRw2pahHwCd5Ec+vwzlbWMoJjh5vQJtibQHdVHQC8A7wUpgxzVXWIqg5p3759BKeuLL+0FJfHgxtweTzkl9oVaGPyJ5XrMbOHNQsZ0wTUNGrox0Au0IVI4zAAACAASURBVB7YBHwE/Aa4U1XdERy7GOgS9LozsDN4A1UtCXr5LPCLyIodnbzMTNIcjsAdQV6mXYE2NksqZ0zTUVNn8STgEPAW8CHwL1WNpoF9NdBbRHrgHRU0EbgxeAMR6aiqu3wvrwS+iOL4EcvNyGD5wIHkl5aSl5lpzULGGBOkpj6Cs32T1J8P5AEPi0hrYD3e0UQv1HRgVa0QkSnA23iHjz6vqp+LyKPAGlVdAtzjm/2sAm+a65tj8J5Cys3IsABgjDEhiGrVZvsQG4mkAIPxpqD+AdBDVZ1xLltIQ4YM0TVr1tS+oTHGmAARWauqQ0Ktq6mP4Eq8dwMXAP3wJp37EHjA99sYY0wzUFMfwc14K/wHgbW+ZwGMMcY0MzUFgglaS7uRiEht2xhjjGnaanqOYIWI3O17gCxARNJE5GLfJPY3xbd4xhhj4q2mO4IxwK3Aq74hoKVAOt4RQMuAp1V1XfyLaBKJJZMzJvHUNHz0GPBb4LcikgqcChxVVXss14RkyeSMSUwRzVmsquWqusuCQHwU7Chg1qpZFOwoaOyi1IslkzMmMUUyVaWJo4IdBYx6eRQut4s0ZxrLJy0nt0tuYxerTiyZnDGJyQJBI8svysflduFWNy63i/yi/IQNBP5kctZHYExiqTUQ+NJEvKKq+xugPEknr3seac60wB1BXve8xi5SvVgyOWMSTyR3BB3wzi72CfA88HYiPjtQUAD5+ZCXB7lN6II7t0suyyctJ78on7zueQl7N2CMSVyR5hoSYDRwCzAEeA2Yp6r/iW/xqqtLrqGCAhg1ClwuSEuD5cubVjAwxph4qynXUKSjhhTY7fupANoCC0Xk8ZiVMo7y871BwO32/s7Pb+wSGWNM0xFJH8E9eJ8g/hZ4DvgfVS0XEQewGW8uoiYtL897J+C/I8jLa+wSGWNM0xFJH8GpwDWqui14oap6ROSK+BQrtnJzvc1BTbGPwBhjGlskgWAp3kljABCRNkBfVf2XqsZlRrF4yM21AGCMMaFE0kfwO7xTVvod9i0zxhjTDEQSCCqlmlZVD/YgWlIoKyhj26xtlBVEM1W1MSbRRFKhf+XrMPbfBdwFfBW/IpmmwBLIGZM8Irkj+CHeKSu/BoqBYcCd8SyUaXyWQM6Y5FFrIFDVPao6UVVPU9XTVfVGVd0TycFFZIyIbBKRLSLycA3bXSsiKiIhH3YwDc+fQA4nlkDOmGYukucI0oHb8E5gn+5frqq31rKfE5gDXIr3TmK1iCxR1Y1VtmsD3AP8K+rSm7ixBHLGJI9Imob+iDff0GXAe0Bn4GAE+w0FtqjqV76J7+cDV4XYbibwOHAsohKbBpORm0G3ad0sCBjTzEUSCHqp6iPAYVV9CRgHZEWwXydgR9DrYt+yABHJAbqo6lsRltcYY0yMRRIIyn2/S0WkP5ABdI9gPwmxLDAM1Zei4mnggVoPJHKniKwRkTV79+6N4NTGGGMiFUkgmCsibYH/BywBNgK/iGC/YqBL0OvOwM6g122A/kC+iBQB5wFLQnUYq+pcVR2iqkPat28fwamNMcZEqsbOYt9V+wHfpDQrgTOjOPZqoLeI9MA79HQicKN/paqW4c1j5D9XPvDfqhpdjmljjDH1UuMdge8p4il1ObCqVvj2fRv4AnhNVT8XkUdF5Mq6HNMYY0zsRfJk8T9F5L+BBXjzDAGgqvvC7xLYZinepHXBy6aH2TYvgrIYY4yJsUgCgf95gclBy5TomokaXcGOgqSbDrKsoMyeA2iCysvLKS4u5tgxGzFtYi89PZ3OnTuTmpoa8T61BgJV7VGvUjUBBTsKGPXyqMAE8csnLW/2wcByBTVdxcXFtGnThu7du+OdBdaY2FBVSkpKKC4upkePyKvuSJ4snhTmhC9HUb5GlV+Uj8vtwq1uXG4X+UX5zT4QhMoVZIGgaTh27JgFARMXIkK7du2Idph9JE1D5wb9nQ6MAj4BEiYQ5HXPI82ZFrgjyOue19hFijt/riD/HYHlCmpaLAiYeKnLv61ImoburnKSDLxpJxJGbpdclk9anlR9BJYryBgTqUgeKKvqCNA71gWJt9wuuUy7aFpSBAE/yxVkwvnmm2+48cYbOfPMMxk8eDC5ubksWrSoXsecMWMGTzzxBADTp0/nnXfeqdNx1q1bx9KlS0Ouy8/PJyMjg5ycHPr06cPw4cN5662mlaHm5ptvZuHChSHXTZ06lZUrVwKwdetWhg0bRu/evbn++utxuVzVti8pKWHkyJG0bt2aKVMqj+S/5JJL2L9/f0zKXGsgEJE3RWSJ7+ctYBPw15ic3RjT4FSV8ePHM3z4cL766ivWrl3L/PnzKS4urrZtRUVFnc7x6KOPcskll9Rp35oCAcBFF11EYWEhmzZtYvbs2UyZMoXly5fX6VwNad++fXz00UcMHz4cgIceeoj77ruPzZs307ZtW+bNm1dtn/T0dGbOnBkIsMG+//3v89vf/jYmZYvkjuAJ4EnfzyxguKqGnVvAGBN7BTsKmLVqFgU7Cup9rHfffZe0tDR++MMfBpZ169aNu+/2tgK/+OKLfPe73+U73/kOo0eP5tChQ4waNYpBgwaRlZXFX/964jrwZz/7GX369OGSSy5h06ZNgeXBV8Vr165lxIgRDB48mMsuu4xdu3YBkJeXx0MPPcTQoUM566yzWLVqFS6Xi+nTp7NgwQKys7NZsGBBje8lOzub6dOn85vf/AaAvXv3MmHCBM4991zOPfdcPvjgAwDee+89srOzyc7OJicnh4MHvQmUH3/8cbKyshg4cCAPP+yt1v7zn/8wZswYBg8ezEUXXcS///3vwHu65557OP/88znzzDMD709VmTJlCn379mXcuHHs2RN6upaFCxcyZsyYwD7vvvsu1157LQA33XQTixcvrrZPq1atuPDCC0lPT6+27sorr+TVV1+t8fOJmKrW+AP0ANKDXrcEute2X7x+Bg8erMYkso0bN0a1/YfbP9SWP22pzp84teVPW+qH2z+s1/l/9atf6dSpU8Ouf+GFF7RTp05aUlKiqqrl5eVaVlamqqp79+7Vnj17qsfj0TVr1mj//v318OHDWlZWpj179tRf/vKXqqp600036euvv64ul0tzc3N1z549qqo6f/58veWWW1RVdcSIEXr//ferqurf/vY3HTVqVOD8kydPDlm2FStW6Lhx4yotKyws1LPPPltVVW+44QZdtWqVqqpu27YtsPyKK67Q999/X1VVDx48qOXl5bp06VLNzc3Vw4cPq6oG3u/FF1+sX375paqqfvTRRzpy5MjAe7r22mvV7Xbr559/rj179lRV1TfeeEMvueQSraio0K+//lozMjL09ddfr1b2SZMm6ZIlSyp9jn7bt2/Xfv361fidhPpMevXqpd9++2215aH+jQFrNEy9GsmoodfxTlXp5/YtOzf05saYWIr38OfJkyfz/vvvk5aWxurVqwG49NJLOeWUUwDvxeKPf/xjVq5cicPh4Ouvv+abb75h1apVXH311Zx00kmA9wq1qk2bNvHZZ59x6aWXAuB2u+nYsWNg/TXXXAPA4MGDKSoqqlP5vXWc1zvvvMPGjSfmvjpw4AAHDx7kggsu4P777+d73/se11xzDZ07d+add97hlltuCZT/lFNO4dChQ3z44Yd897vfDRzj+PHjgb/Hjx+Pw+Ggb9++fPPNNwCsXLmSG264AafTyRlnnMHFF18cspy7du3CnzQzuMx+dRntc9ppp7Fz507atWsX9b7BIgkEKeqdWAYAVXWJSFq9zmqMiVishz/369ePN954I/B6zpw5fPvttwwZciLxb6tWrQJ/v/LKK+zdu5e1a9eSmppK9+7dA09F11Z5qSr9+vWjoCB0k1aLFi0AcDqdde6PKCws5JxzzgHA4/FQUFBAy5YtK23z8MMPM27cOJYuXcp5553HO++8g6pWK7/H4yEzM5N169bVWF7/e/OLpBJv2bJl4HM79dRTKS0tpaKigpSUFIqLiznjjDMie8NBjh07Vu291kUkfQR7g5PEichVwLf1PrMxJiL+4c8zR86MyVPxF198MceOHeN3v/tdYNmRI0fCbl9WVsZpp51GamoqK1asYNu2bQAMHz6cRYsWcfToUQ4ePMibb75Zbd8+ffqwd+/eQCAoLy/n888/r7F8bdq0CbTh12bDhg3MnDmTyZO9GXBGjx4d6C8AAhX6f/7zH7KysnjooYcYMmQI//73vxk9ejTPP/984L3v27ePk08+mR49evD6668D3sp+/fr1NZZh+PDhzJ8/H7fbza5du1ixYkXI7c455xy2bNkCeAPHyJEjA/0ML730ElddFWoCx/BUld27d9O9e/eo9gslkkDwQ+DHIrJdRLYDDwE/qPeZjTERi+XwZxFh8eLFvPfee/To0YOhQ4dy00038YtfhJ5m5Hvf+x5r1qxhyJAhvPLKK5x99tkADBo0iOuvv57s7GwmTJjARRddVG3ftLQ0Fi5cyEMPPcTAgQPJzs7mww8/rLF8I0eOZOPGjWE7i1etWhUYPjp58mRmz57NqFGjAJg9ezZr1qxhwIAB9O3bl9///vcAPPPMM/Tv35+BAwfSsmVLLr/8csaMGcOVV17JkCFDyM7ODozMeeWVV5g3bx4DBw6kX79+lTrHQ7n66qvp3bs3WVlZ/OhHP2LEiBEhtxs3bhz5+fmB17/4xS946qmn6NWrFyUlJdx2220ALFmyhOnTT+Tm7N69O/fffz8vvvginTt3DjR9rV27lvPOO4+UlEgadmomodqqQm4o0tq3fWShOk6GDBmia9bYlAUmcX3xxReBpgyTXC688ELeeustMjPr/6T/vffey5VXXhkIgsFC/RsTkbWqWm3iL4jsOYKfi0imqh5S1YMi0lZEflrXwhtjTLJ68skn2b59e0yO1b9//5BBoC4iaRq6XFVL/S/UO1vZ2Jic3RhjksiwYcMYMGBATI51xx13xOQ4EFkgcIpIoKtcRFoCLWrY3kSprKCMbbO2UVZQ1thFMcYkoUh6Gf4ELBeRF/BOSHMrCZR5tKmzeQOMMY0tkuyjj4vIBuASQICZqvp23EuWJGzeAGNMY4so+6iq/kNV/1tVHwAOicicOJcrafjnDcCJzRtgjGkUEQUCEckWkV+ISBHwU+DfcS1VEvHPG9BjZg9rFjINRkR44IEHAq+feOIJZsyYEfVxioqK+POf/xzDkkXm6NGjjBgxArfbDXgfyOrduze9e/fmpZdeCrnP66+/Tr9+/XA4HAQPQf/000+5+eabG6LYTVbYQCAiZ4nIdBH5AvgNUIz3OYKRqvrrSA4uImNEZJOIbBGRahlLReSHIvKpiKwTkfdFpG+d30kCs3kDTENr0aIFf/nLX/j22/olCahLIPBX3vXx/PPPc8011+B0Otm3bx8/+clP+Ne//sXHH3/MT37yk5B5+vv3789f/vKXQBpov6ysLIqLi2M2rDMR1XRH8G+801J+R1Uv9FX+EX+DIuIE5gCXA32BG0JU9H9W1SxVzQYeB56KqvTGJImCsjJmbdtGQVlsRpalpKRw55138vTTT1dbF00q54cffphVq1aRnZ3N008/jdvt5n/+538499xzGTBgAH/4wx8A74QyI0eO5MYbbyQrKwuAp556iv79+9O/f3+eeeYZwJujPzjH/owZM3jyySerlfGVV14JpGR4++23A0ny2rZty6WXXso//vGPavucc8459OnTJ+Tn8Z3vfIf58+dH8xE2KzV1Fk8AJgIrROQfwHy8ncWRGgpsUdWvAERkPnAVEEgNqKoHgrZvhXdUkjEmSEFZGaPWr8fl8ZDmcLB84EByM+p/9zh58mQGDBjAgw8+WGn5vffey3333ceFF17I9u3bueyyy/jiiy944oknmDNnDhdccAGHDh0iPT2dxx57jCeeeCIwS9jcuXPJyMhg9erVHD9+nAsuuIDRo0cD8PHHH/PZZ5/Ro0cP1q5dywsvvMC//vUvVJVhw4YxYsQIJk6cyNSpU7nrrrsAeO2116pV6i6Xi6+++iqQY+frr7+mS5cugfWdO3fm66+/juqzGDJkCI899li1zyJZhA0EqroIWCQirYDxwH3A6SLyO2CRqi6r5didgB1Br4uBYVU3EpHJwP1AGhAyf6uI3AncCdC1a9daTmtM85JfWorL48ENuDwe8ktLYxIITj75ZCZNmsTs2bMrZbCMJpVzVcuWLWPDhg2BZGplZWVs3ryZtLQ0hg4dSo8ePQB4//33ufrqqwNZTq+55hpWrVrFPffcw549e9i5cyd79+6lbdu21f7Pf/vtt5VSNMQipbM/nXOyqrWzWFUPq+orqnoF0BlYB0QyQ1mob6LaN6aqc1S1J95kdv8vTBnmquoQVR3iz+dtTLLIy8wkzeHACaQ5HOTFIE+N39SpU5k3bx6HDx8OLPOncl63bh3r1q3j66+/pk2bNjz88MM899xzHD16lPPOOy8wc1cwVeXXv/51YN+tW7cG7giCU1vXlOPs2muvZeHChSxYsICJEydWWx+czhm8dwA7dpy45qxLSudYpXNOVFFNXq+q+1T1D6oaeuaFyoqBLkGvOwM1hdz5eO88jDFBcjMyWD5wIDN79IhZs5DfKaecwnXXXVdpvtxoUjlXTRl92WWX8bvf/Y7y8nIAvvzyy0pBxm/48OEsXryYI0eOcPjwYRYtWhTIXjpx4kTmz5/PwoULA1M5Bmvbti1utzsQDC677DKWLVvG/v372b9/P8uWLeOyyy6L6nP48ssv6d+/f1T7NCdRBYIorQZ6i0gP30Q2E4ElwRuISO+gl+OAzXEsjzEJKzcjg2ndusU0CPg98MADlUYPRZPKecCAAaSkpDBw4ECefvppbr/9dvr27cugQYPo378/P/jBD0JOODNo0CBuvvlmhg4dyrBhw7j99tvJyckBvBPnHDx4kE6dOlWazSzY6NGjef/99wFvMHvkkUcCndvTp08PzK52++23B4aKLlq0iM6dO1NQUMC4ceMqBYsVK1Ywbty4GHyaiSniNNR1OrjIWOAZwAk8r6o/E5FH8c6duUREfoX3ieVyYD8wRVVrnLXC0lCbRGdpqOuvsLCQp556ij/+8Y/1Ptbx48cZMWIE77//fkxy+zcF0aahjuu7VtWlwNIqy6YH/X1vPM9vjGmecnJyGDlyJG63G6fTWa9jbd++nccee6zZBIG6SN53boxJaLfeemtMjuN/IjmZxbOPwBhjTAKwQGCMMUnOAoExxiQ5CwTGGJPkkiYQ2HSQxlS2aNEiRCTkE8J+N998cyBdRCSKiopqfTAreJt169axdOnSGrcPZdeuXVxxxRWB17NmzaJXr1706dOHt98OPW/Wb37zG3r16oWIVHpu4q233uL//u//oi5Dc5IUgcA/HeTWR7ayftR6CwbGAK+++ioXXnhho2bdrGsgeOqppwKTt2/cuJH58+fz+eef849//IO77rorZKrrCy64gHfeeYdu3bpVWj5u3DiWLFnCkSNH6vYmmoGkCAShpoM0JpHE+o720KFDfPDBB8ybN69SIFBVpkyZQt++fRk3bhx79uwJrHv00Uc599xz6d+/P3feeWcgX9DatWsZOHAgubm5zJlzYvLCcCmp/VwuF9OnT2fBggVkZ2ezYMECPv74Y84//3xycnI4//zz2bRpU8jyv/HGG4wZMwaAv/71r0ycOJEWLVrQo0cPevXqxccff1xtn5ycnEDG0mAiQl5eXiCDajJKikBg00GaRBaPO9rFixczZswYzjrrLE455RQ++eQTwNtctGnTJj799FOeffZZPvzww8A+U6ZMYfXq1Xz22WccPXo0UHHecsstzJ49m4KCgkrnmDdvXiAl9erVq3n22WfZunVrYH1aWhqPPvoo119/PevWreP666/n7LPPZuXKlRQWFvLoo4/y4x//uFrZt27dStu2bWnRogUQuzTUq1atimqf5iQpHijzTwdZml9KZl6mzQRmEkqoO9r6/ht+9dVXmTp1KuBN8vbqq68yaNAgVq5cyQ033IDT6eSMM87g4otP5JdcsWIFjz/+OEeOHGHfvn3069eP4cOHU1payogRIwD4/ve/z9///ncgfErqs846K2y5ysrKuOmmm9i8eTMiEkheF2zXrl0EZyG2NNT1lxSBALzBwAKASUT+O1qPyxOTO9qSkhLeffddPvvsM0QEt9uNiPD4448DoSvRY8eOcdddd7FmzRq6dOnCjBkzOHbsGKoattL1p6Sumgm0qKgobNkeeeQRRo4cyaJFiygqKiIvL6/aNpaGOvaSomnImETmv6PtMbMHA5cPrPcFzcKFC5k0aRLbtm2jqKiIHTt20KNHD95//32GDx/O/Pnzcbvd7Nq1ixUrVgAEKt5TTz2VQ4cOBa7yMzMzycjICGQCfeWVVwLniSQlddU01mVlZXTq1AmAF198MWT5zzrrrErB5Morr2T+/PkcP36crVu3snnzZoYOHRrVZ2JpqI0xTV5GbgbdpnWLyV3tq6++ytVXX11p2YQJE/jzn//M1VdfTe/evcnKyuJHP/pRoMknMzOTO+64g6ysLMaPH8+5554b2PeFF15g8uTJ5ObmVrqqjiQl9ciRI9m4cWOgs/jBBx9k2rRpXHDBBWEnuW/VqhU9e/Zky5YtgDdt9XXXXUffvn0ZM2YMc+bMCSSiGzt2bKDJZ/bs2XTu3Jni4mIGDBjA7bffHjimpaGOYxrqeLA01CbRWRrq+lu0aBFr167lpz/9ab2P9c0333DjjTeyfPnyGJSsaWhSaaiNMSYerr76akpKSmJyrO3bt/Pkk0/G5FiJygKBMSYhBTft1EdwM1eysj4CY4xJchYIjDEmyVkgMMaYJGeBwBhjkpwFAmOSUHFxMVdddRW9e/emZ8+e3HvvvbhcLgDy8/PJyMggJyeHPn36MHz48EoJ2WbMmEGnTp3Izs4O/JSWVk7k6PF4uOeee+jfvz9ZWVmce+65lfIMhdK9e/dK6aFNw4lrIBCRMSKySUS2iMjDIdbfLyIbRWSDiCwXkW6hjmOMiR1V5ZprrmH8+PFs3ryZL7/8kkOHDvG///u/gW0uuugiCgsL2bRpE7Nnz2bKlCmVxtnfd999rFu3LvCTmVk57cWCBQvYuXMnGzZs4NNPP2XRokXVtonXe/N4PHE/T3MTt0AgIk5gDnA50Be4QUT6VtmsEBiiqgOAhcDj8SqPMYmsoABmzfL+rq93332X9PR0brnlFgCcTidPP/00zz//fMic/NnZ2UyfPp3f/OY3EZ9j165ddOzYEYfDW8V07tyZtm3bAt4nm7Oysujfvz8PPfRQtX0feughfvvb3wZez5gxIzDO/5e//GUgrbV/MpmioiLOOecc7rrrLgYNGlQp75CJTDzvCIYCW1T1K1V1AfOBq4I3UNUVqur/l/cR0DmO5TEmIRUUwKhR8Mgj3t/1DQaff/45gwcPrrTs5JNPpmvXroG0DVUNGjSo0kxmTz/9dKBZaOTIkdW2v+6663jzzTfJzs7mgQceoLCwEICdO3fy0EMP8e6777Ju3TpWr17N4sWLK+07ceJEFixYEHj92muv8d3vfpdly5axefNmPv74Y9atW8fatWtZuXIlAJs2bWLSpEkUFhZWm3jG1C6egaATEByai33LwrkN+HuoFSJyp4isEZE1e/fujWERjWn68vPB5QK32/s7P79+xwuXMbS2TKLBgpuG/InpgnXu3JlNmzYxa9YsHA4Ho0aNYvny5axevZq8vDzat29PSkoK3/ve9wKVuV9OTg579uxh586drF+/nrZt29K1a1eWLVvGsmXLyMnJCQSmzZs3A9CtWzfOO++8un4kSS+eTxaH+hcVMrGRiPwXMAQYEWq9qs4F5oI311CsCmhMIsjLg7Q0bxBIS/O+ro9+/frxxhtvVFp24MABduzYQc+ePUOmbigsLIw6P1KLFi24/PLLufzyyzn99NNZvHgxo0aNimjfa6+9loULF7J7924mTpwIeIPRtGnT+MEPflBp26KiIlq1ahVV2Uxl8bwjKAa6BL3uDFSb+UFELgH+F7hSVY/HsTzGJKTcXFi+HGbO9P7Oza3f8UaNGsWRI0d4+eWXAe+Ukg888AA333wzJ510UrXtN2zYwMyZM5k8eXLE5/jkk08CWT89Hg8bNmygW7duDBs2jPfee49vv/0Wt9vNq6++GshwGmzixInMnz+fhQsXcu211wLetNbPP/88hw4dArwzkwVPpWnqLp53BKuB3iLSA/gamAjcGLyBiOQAfwDGqKp9o8aEkZtb/wDgJyIsWrSIu+66i5kzZ+LxeBg7diw///nPA9usWrWKnJwcjhw5wmmnncbs2bMrXc0//fTT/OlPfwq8Xrx4caX5gPfs2cMdd9zB8ePea7uhQ4cyZcoU0tPTmTVrFiNHjkRVGTt2LFddVanrEPDetRw8eJBOnTrRsWNHAEaPHs0XX3xBru+DaN26NX/6058CKadN3cU1DbWIjAWeAZzA86r6MxF5FFijqktE5B0gC9jl22W7ql5Z0zEtDbVJdJaG2sRbk0pDrapLgaVVlk0P+vuSeJ7fGGNM7ezJYmOMSXIWCIxpBIk2M6BJHHX5t2WBwJgGlp6eTklJiQUDE3OqSklJCenp6VHtZzOUGdPA/BOo28ORJh7S09Pp3Dm6JA0WCIxpYKmpqfTo0aOxi2FMgDUNGWNMkrNAYIwxSc4CgTHGJLm4PlkcDyKyF9jme5kBlIXYLNzyU4GmOgVSuDI39nHrsn+k+9S2XV3XJ9r3H6/vPhbHjnb/WH33tW1Tl3XJ/v13U9X2IbdS1YT9AeZGuXxNY5c52vfS2Mety/6R7lPbdnVdn2jff7y++8b4/mP13de2TV3W2fcf/ifRm4bejHJ5UxavMtf3uHXZP9J9atuurusT7fuPZ3kb+vuP1Xdf2zZ1XdcUNfr3n3BNQ/UhIms0TNIl0/zZ95/cskM2NgAABjBJREFU7PsPL9HvCKI1t7ELYBqVff/Jzb7/MJLqjsAYY0x1yXZHYIwxpgoLBMYYk+QsEBhjTJJL6kAgIq1E5CUReVZEvtfY5TENS0TOFJF5IrKwsctiGpaIjPf9v/+riIxu7PI0tmYXCETkeRHZIyKfVVk+RkQ2icgWEXnYt/gaYKGq3gHUOFeySQzRfP+q+pWq3tY4JTWxFuV3v9j3//5m4PpGKG6T0uwCAfAiMCZ4gYg4gTnA5UBf4AYR6Qt0Bnb4NnM3YBlN/LxI5N+/aV5eJPrv/v/51ie1ZhcIVHUlsK/K4qHAFt8VoAuYD1wFFOMNBtAMP4tkFOX3b5qRaL578foF8HdV/aShy9rUJEvl14kTV/7gDQCdgL8AE0TkdyTeY+kmciG/fxFpJyK/B3JEZFrjFM3EWbj/+3cDlwDXisgPG6NgTUmyzFAmIZapqh4GbmnowpgGF+77LwGSvhJo5sJ997OB2Q1dmKYqWe4IioEuQa87AzsbqSym4dn3n7zsu49AsgSC1UBvEekhImnARGBJI5fJNBz7/pOXffcRaHaBQEReBQqAPiJSLCK3qWoFMAV4G/gCeE1VP2/Mcpr4sO8/edl3X3eWdM4YY5Jcs7sjMMYYEx0LBMYYk+QsEBhjTJKzQGCMMUnOAoExxiQ5CwTGGJPkLBCYRiciKiJPBr3+bxGZEaNjvygi18biWLWc57si8oWIrKiyvLs/LbKIZIvI2BieM1NE7gp6fYbNrWDqwgKBaQqOA9eIyKmNXZBgvhTGkboNuEtVR9awTTYQVSAQkZrygWUCgUCgqjtVNe5BzzQ/FghMU1ABzAXuq7qi6hW9iBzy/c4TkfdE5DUR+VJEHhOR74nIxyLyqYj0DDrMJSKyyrfdFb79nSLySxFZLSIbROQHQcddISJ/Bj4NUZ4bfMf/zJfGGBGZDlwI/F5EfhnqDfrSGzwKXC8i60Tket8Mec/7ylAoIlf5tr1ZRF4XkTeBZSLSWkSWi8gnvnP7U2g/BvT0He+XVe4+0kXkBd/2hSIyMujYfxGRf4jIZhF5POjzeNH3vj4VkWrfhWm+kiX7qGn65gAb/BVThAYC5+DNQf8V8JyqDhWRe/GmGZ7q2647MALoCawQkV7AJKBMVc8VkRbAByKyzLf9UKC/qm4NPpmInAH8AhgM7MdbSY9X1UdF5GLgv1V1TaiCqqrLFzCGqOoU3/F+DryrqreKSCbwsYi849slFxigqvt8dwVXq+oB313TRyKyBHjYV85s3/G6B51ysu+8WSJytq+sZ/nWZQM5eO/ENonIr4HTgE6q2t93rMyaP3rTnNgdgWkSVPUA8DJwTxS7rVbVXap6HPgP4K/IP8Vb+fu9pqoeVd2MN2CcDYwGJonIOuBfQDugt2/7j6sGAZ9zgXxV3evLYfMKMDyK8lY1GnjYV4Z8IB3o6lv3T1X1T7IiwM9FZAPwDt58+qfXcuwLgT8CqOq/gW2APxAsV9UyVT0GbAS64f1czhSRX4vIGOBAPd6XSTB2R2CakmeAT4AXgpZV4LtgEREB0oLWHQ/62xP02kPlf9tVE2op3sr1blV9O3iFiOQBh8OUL1Ru+/oQYIKqbqpShmFVyvA9oD0wWFXLRaQIb9Co7djhBH9ubiBFVfeLyEDgMrx3E9cBt0b0LkzCszsC02T4roBfw9vx6leEtykGvNNLptbh0N8VEYev3+BMYBPebJQ/EpFUABE5S0Ra1XKcfwEjRORUX0fyDcB7UZTjINAm6PXbwN2+AIfI/2/n7lEiCIIwDL+tB/AAggcwU7yANzH0CgZGG5sKooGJgYGBkeGK4SIL/gR6AkHwDGXQJSwyg2sm1vvkPXQn/XVXNdO2RsatAe8ZArv0E/zQ9xbd0QOELAlt0Nc9KEtOKxFxBRwC20utSP+CQaC/5ghYfD10St98Z8D3k/KyXukb9g2wnyWRM3pZZJ4N1hN+uCFHxBtwAEyBB2AeEde/mMcU2PxqFgMTerA95hwmI+MugJ3W2j19c3/J+XzQexvPA03qY2C1tfYEXAJ7WUIbsw7cZpnqPNepIvwNtSQV541AkoozCCSpOINAkoozCCSpOINAkoozCCSpOINAkoozCCSpuE/084eKJz2geQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(range(len(all_accuracy[0])), all_accuracy[0], '.', color='g', label='Gradient Descend (0.1)')\n",
    "plt.plot(range(len(all_accuracy[1])), all_accuracy[1], '.', color='c', label='Nesterov (0.1)')\n",
    "plt.plot(range(len(all_accuracy[2])), all_accuracy[2], '.', color='m', label='Adadelta (0.1)')\n",
    "plt.plot(range(len(all_accuracy_results_ode)), all_accuracy_results_ode, '.', color='b',label='ODE Solver')\n",
    "plt.semilogx()\n",
    "plt.xlabel('Number of Iterations')\n",
    "plt.ylabel('Accuracy (Whole Dataset)')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de3xU1bnw8d8zISHIJSCgICAgIMotASIaLxgEFaEiXlrRnuKdtkK9vhXpOXKotEVtFcsRbb3LqRUUCkWLlWMkAhorYAJqLBchQIAIBBLuJMw87x97ZpwkM8kkmcltnq+ffCazZ1/WbMN69l5r7WeJqmKMMSZ2ueq7AMYYY+qXBQJjjIlxFgiMMSbGWSAwxpgYZ4HAGGNinAUCY4yJcc3quwDV1aFDB+3Ro0d9F8MYYxqVdevW7VfVjsE+a3SBoEePHqxdu7a+i2GMMY2KiGwP9Zk1DRljTIyzQGCMMTHOAoExxsQ4CwTGGBPjLBAYY0yMa3SjhoxprLKKi8ksKqJ9fDyFpaX+1/S2bUlLSqrv4pkYZoHAmAgKVtlnHz5MQUkJ7x84QIkqgYnfBee2/NoOHXikWzd/QPDtJ9wgEbg+wLyCAgAmdupUJ0Ema2cWT33yFJ/lf8ahk4dQlDNbncm0S6cxaeikqB+/qcjKgsxMaN8eCgud1+xs57PBg51l6emQlhbZ40pjm48gNTVV7TkC05D4KuGiU6eYnZ/PKW9lL0B1/nXFA3d17szg1q15YMsWTno8AJwRH8/p8fHc37Urk846q8J2L+7ezZTNmyn1/lsuf9zhSUk8cc45EQ0IL657kWc/e5bvjn7H0ZKjnHSfLPP52LVjuXXVrSQdTyK+fTwDZgzgrEkVy97Y+Sru9HTnfWAlHmxZqNf0dPjyS5gyBU6dAlUQcV4DuVzQvDlkZFQ/GIjIOlVNDfqZBQJjQit/pR3sav+1ggJKVfFE6JiVBZDWLqdbL16EBJeLo243hz1VHzlehI9TUqoVDHyV/cETBwEocZdQ6i7F7XFz7NQxAPrt7MfNn9zM+TvP57TS03Dj5rSS04gjrtx3Es7987mNKhgEVvJpaRUr/Xnz4LXXnIo7Ls6puEtLweNxKuxmzcou81Xs5V9967rdzk9V4uJg5kyYNq1636eyQGBNQ8Z4BWteea2ggFOqxIkg4K/wfZV1da/6xfsaJ8LNHTsyf+9eyv/br2x/ZSr9cGoNr1JVMouKQgaC8pX+0ZKjHC45XGE935V+h8MdcHlcFSr8QOL/to49r+yp10BQvmIPtiywaeaBB6CkBBIS4Nlnv3/vq/RLSr6/Yvf9bwl8X1padlmoV9+65a/JQ90RJCR8H4wixQKBMThBYOT69ZR4PP5KP7A93+P9F+l7X/61vD6JiWw9ccJfyfsChuL0CVzcpg1H3W6ubd/ev827hYUVgkKkuICiwzv4+XtPkLsvl33H9tG8WXMOHj/I4ZLDHDh+oMz6gVf6bU60Ic7tVPjBKv7yFX4oCWcl1Pp7hFJVJQ8wcuT3FXtGRsVlgZW9iFNBezzO+0WLnFe3u2KlLwLx8bW/I/Ct63LBQw9B27Z110dggcDEvKziYmbk5XHS48FDxUpfcJpWgt0RuIBmIvRu0YLcY8f8+9x84kSZYwQGDA+wsrjY/765CCtSUrimfXvu3bSpxsFAgPNOO42O8fFsPVRAvqc5vnsQD/DU3uOweSUcyq2w7aOLHmV47nDiPHG4tHpX+qEEdouLSzj7kbOr83WCClXhV1XJ33bb9xV5SYmzDyi7LLCyd7m+v/JPSIAbb4RVq8reEfiahO68EyZOdPZXmz4C37rRqOirYoHAxKTA0T2+jlkP31fsAv4moTs7dWJip05AxT4C3+svNm+ucVlOqjIjL48ZPXqwavBg5hUU8MWRI6w9fBgPEAfc07kzBSUlfHboEEfdbn8fQfGpUxz3BS71sLPgE75pNRAk0dm5BFbazSApBQ7lcvf/3c34z8eTcCoBUanRlb5W0SimKKVxpRT0LmD8a+NJSqtdZ3WwCj8tzak8q6rkwdnGt62v4g1cFljZ++4QAq/ABw6s2DFcvtIOVoFXp1Kv6wDgY4HAxJzAZiARweO9yncBo9q1Y4Y3zXngEEwoGwR8wzp9dxOnAhpzfVVqYHNQMxHGnH46AP8oLKS0XJn+7+BBPioqYm6fPrzQt2+ZMia4XGWGgWbtzGLe+nkUHCkgz3MaOWdOBFcCIBxpNQjE9X0A8Jbr7hfh2neh+YmJxLnvqHbFH6rS9+BBUU42O4kLFy51cSruFJs7b+alUS+R2y2XP//gzyQNDT8IBLvqh+AVflqas15VlfzEic5P+f1mZJRdFljZl6+U09KqrvQbKxs1ZJqsYCN+0tu2JbOoiMe2bcONU/nHeYNBgstFRnIyUHlHse+uYczpp/P+gQNlljd3uXi2d29/sAg8buAzAvMKCsgtLmDrsUPk09pf5sDRPWXKfyiXeevnkbsvl1U7VpWtmPs8AJ2vdQKA99/z2PfgxkXQ9gC0OuKmmVZd8Vd2ha/e/wBONDvB3nZ7WXThIv6R+o+Q27RJaMPvr/p9tZ4jCHXVH85nVXUExzobNWRiTrDO31Peyv7Z3r1JcLn8V9vP9u5N9mFnhMyXR4/ywJYtnPB4QnYUe3A6kpcUFvqPF3g3UX5kTvkreYA2iW3IyppNaatzIXk2iNNbeErVuRPxVvwASxLb8PSnT+PWEL0H3y2HTmO4+0XhmveFFscgsTSwUneCQDgVv9vbQ6GiHI8/zpHTjrCl0xYWXLKA3G5O30KLZi1oEd8CgNNKT0NRmrma4fa4OaWnOC3+NCYNncSTo54kKwtmzQq/Mg511Q/Oa/kreP85Tqv6Ct6EZoHANEmZRUWUeDy4KVuRl3g8FJaWkpGc7G/6AXjju++cpqKCAjwBo4Uq6ygmYJ3mLleZIJC1M4vMvEzan9ae9ze/T3ZBNjuKd1SsfA99DZv/6FzVE4cCf961gz9tngPxbaA4J2jnLpR9aMvlVhLdvm/pK1d4Fb/b5eZkwkneHfouL1/5cpnPWyc4dystE1oyvut4Hrn4EdK6hVe7VnYFH0qoZh4fq9yjwwKBaZLS27b1X/XHlev83XHiBF8ePVqh8vcALl9TUCUdxb6HyHz7G9NaoGA5T70/G4ADxw+weudqPBrmI2YF/4DWffzNO6ou6HO/dwxjKWx4GA7lMnbtWG787EZanWhF85LmtC5pXWFXVVX+bty449ycjC9b8Z/e4nQS4hLoRCcSmyWS0imlWpV+MJVd3YdS2VW/iR4LBKbJKJ/nx9dWXz7fz0t79oSs/BPKtfGXaeY5lMu8L5xO2gvjO7E/oSsJh3NZunk+nto+V/zdcjhztNPpKy7GvgdjlglxJdC6eBatDlOjit83cqd8xQ/QqVUn+rXox/0X3l+mHd/Xts7ZQLeaf6Wqru5Dsav+uhfVzmIRGQ38EaeR8mVVfaLc52cDbwBtves8qqrLKtundRY3btVNphbudr4+gcBhoL6O2we2bAk6Qiiwk/jZ3r3J3v8tBfkf0MlTyODOg8nek03BEaf56MDxA6zesbr2FX4Qviv95pzB4aQWnLHXRdvDwf9dhtPccyLuBKXxpWw9cysvjXqJYwOP0TaxLSdPnaRjy47069CPickTg17t16Q5pzLWYdtw1EtnsYjEAXOBK4F8YI2ILFXVwAbP/wLeVtUXRKQfsAzoEa0ymeirrMIuPyQyIzm5ymDgG2Hja4oJtZ2vT8BXTXtw+gMW7dvn7ytwqSKAoMShpB1fx/7SEs6NO863uf/gpco6ZGtBEK44eAXjvh5Hr8JenNp/isMtD7P/+H667O9Ch6Md/Ot23g++dv5gwzmDVfwn405yPOE4Ra2K2DBmA19c/gX7ju2jb4e+vHzxy9Vq3qlJc05l7Oq+cYhm09AwYIuqbgUQkfnAdUBgIFCgjff3JGB3FMtjoqyqij6wA7fE46k0903g/gJH8ITaztcnUPbBMDitaC0ueuJRADdsmQtxrSgtzmaltxM2eFds9bhwcWn3Szk98XT/sk6bOjH0s6H03t4bvqFMD3MSSXShi/99uJU+wOGEw5xMOMmRFkfYMGYDeaPz6NSqExOTJ3JHtztq9T1q2pxjGrdoBoIuwM6A9/nAheXWmQEsF5FfAC2BUVEsj6mFYFf65ZcFq+jh+3H0gR24CS6Xf5x9KL79BY7gCdyu/PGf7RzHot3b6JjQnM2H9pL979dYWrQBT5vzICkFrWQETriCVvjeSjitWxrFWcUUzCvgaO5RDq0+RGUtSYGVf6hK/7vW3+FxeYh3x3OkxREWXbiI91PfZ9x543jk4kdqXfGXZ521sSmagSDYY4rl/9pvAV5X1adFJA34XxEZoFp2uIWITAImAZx9du3zlZjqCXalD1RYVr6ibx8fX2GdjOTksPsIyo/88Y3g8T1s5dt3M4Frjn7I+1/8gVJ3KR48CPJ95XooN6wAIAgucXFt32u5pvc1ZfoIoGyFX15xVjFf3vclhUsLK638g/E9rFXYqpCDrQ7S6ngrTiacLPPAliBc1v0y+nXox+rk1ZCfRuZfgPTIV9bWnBN7ohkI8ik75qArFZt+7gJGA6hqlogkAh2AvYErqeqLwIvgdBZHq8AmuFBX+uWXTevevUxFH2y7ad27h91JnJaUFDJwZBYV+ZuB3O5TLCnYBgGTo4S6wo53xXPX4LvKdAZ3atWJwZ0HU3iskPQe6dUeMvnt1G/Z+fudYeWjTuyTiKuZi/iO8SjKqX2n2JS0iVn9Z/kf2CovTuJ4fuzz/pE9ke7QNSaagWAN0EdEegK7gAnAreXW2QGMBF4XkfOBRGBfFMtkaiBUk06wZWlJSWUq7Oo0BQVTfn8+7Uvy8bhPOE/k6innwasALnHRzNWMO1Pu9Ff6QMgr+pr6duq37HxqZ+gVXNByUEtcCS4639U5aD5+9043m17fVOFOIk7iuLbvtRXG80e6Q9eYqAUCVT0lIlOAD3CGhr6qql+LyOPAWlVdCjwMvCQiD+JcT92ujS35UQwIdWVeVTNPZVf0tfHi5k/4/df/B9+t8z99K4e+IT4uwV/x1/Tqvjp2v7jbuRMozwVJlyZxWr/T6DSxU5VZN9O6pTF3zFymLJuCW93+ABYqaFmHrok0Szpn6k1Nnil4cfMn/HTHYXDFg6cU2fBL4lxxDDnvTu46ZyiT+lwS5VI7irOKyRmeg54q++8naXgS5zxxTo1SLvvSUoQTwGx8vqkuSzpnGpzqPlPgqySXHGsJCf1B4kCUs3rfwf42Q1in8OUeNwPPKI7oJO2hFMwrKBsEBLr9shu9nuxV432mdUsL+w7GOnRNJFkgMPWiOs8UZO3MYuS8kZS4S5CkATDgCRAFPcUFXVJ591D4zybUlm946J5X9ny/MA7Ofb5xTcxuTCALBKZeVOeZgsy8TErcJbjVTVzxV4w/uZpjLftw41m9GXjGAD4IuLOoSYd0uIqzilk/cj2eE54y81h2vid4J7AxjYUFAlMvynckA8zavj1of0F6j3QS4hIocZeQEJfAIwPHlmlCiUaHdDAF8woqBAFXootOEztF7ZjG1AXrLDb1Lpz+gup0pEZDcVYxOek5aIn330s8dL6rc1ijgoxpCKyz2DRolfUXBAaAaZdNq7cyFswrQH2zfokTBPq+0LfeymNMJFkgMGXUNE10bYTqLwjsJE6ISyBjYka93Q0UvFrgbxKSBLHmINOkWCAwfjVJEx0JoR48C+wkLnGXkJmXWS+BoPzdQKc7rDnINC0WCIxfddNER1KwVBLlO4nTe6TXSVkC2d2AiQUWCIxfddNER1tatzQyJmbUaydxUWYR6ra7AdO0WSAwftHKDVSrMlXjadtoaJveFleCC0+JB1eCDRU1TZMFAlNGqGyfdam+h4qWd+ZtZwLYUFHTZFkgMA1K1s4s0hdPprR1f+Kz3yHz+rn1Fgz8TxLb3YBp4iwQmAZl3ra1lPT/HbjiKfGUMm/b2noLBEWZRXhKPOAGT4mHoswiuyMwTZKrvgtgTBltU5wU0xLnTDrTNqXeihLfPh5xCbjAleCibXr9dp4bEy1V3hGIyBnAJcBZwHHgK5yJZao5M6sxVZvYfRCvHcymRJWEuGZM7D6oXspRnFXMlge2oG5FXELvZ3vb3YBpskIGAhEZATwKnA5k48wjnAiMB3qJyELgaVU9VBcFNbEhLSmJFSmD633kkr9ZyAMqSmlhab2Uw5i6UNkdwRjgHlXdUf4DEWkG/AC4ElgUpbKZGNUQRi6VHzZqzUKmKQsZCFT1lwAi0lNVt5X7uJuqLolqyYypJ8VZxRRlFtH72d6UFpbSNr2tNQuZJi2cUUOLgCHlli0Ehka+OCZWNZRnB8oPGU3OSLYgYJq8yvoIzgP6A0kickPAR21w+gqMiYiGkmUUbMioiU2VDR/ti9MP0Ba4NuBnCHBPODsXkdEislFEtojIo0E+ny0iOd6fTSJSVP2vYBq7YFlG64uvb4A4GzJqYkdlfQR/B/4uImmqmlXdHYtIHDAXp0M5H1gjIktVNTfgGA8GrP8LYHB1j2Mav4aQZTSQpZQwsSacPoJCEckAzlTVASIyCBinqr+pYrthwBZV3QogIvOB64DcEOvfAvx3mOU2TUhDyDIKllLCxK5wnix+CZgGlAKo6gZgQhjbdQF2BrzP9y6rQES6Az2Bj0J8PklE1orI2n379oVxaNPYpHVLY9pl0+q1ozhY/4AxsSCcQHCaqn5ebtmpMLaTIMs0xLoTgIWq6g72oaq+qKqpqprasWPHMA5tTPVZ/4CJVeE0De0XkV54K3ERuQnYE8Z2+UC3gPddgd0h1p0ATA5jn8ZElfUPmFgUTiCYDLwInCciu4BtwH+Esd0aoI+I9AR24VT2t5ZfSUT6Au2AandIm5qpjwnqQ5algT4/YP0DJpZUGQi8nb2jRKQl4FLVw+HsWFVPicgU4AMgDnhVVb8WkcdxktYt9a56CzBfVUM1G5kIqq8J6oOWxZ4fMKZBqLKPQETuF5E2wDFgtoh8ISJXhbNzVV2mqueqai9V/a132fSAIICqzlDVCs8YmOgINkF9vZWlgTw/UJxVzIkdJ5BmYv0DJiaF01l8pzfD6FXAGcAdwBNRLZWJGt8E9XFQ7xPU+54fiJO4ent+wNcktOelPaDQ+Z7OllbCxJxw+gh8o3/GAK+p6noRCTYiyDQCDWmC+obw/EBgk5CiJJ6daEHAxJxwAsE6EVmOM85/moi0BmxSmgYqnI7g+kzzXL5z2PdTXyzdtDHhBYK7gBRgq6oeE5H2OM1DpoFpSB3BwTSkzuFANmTUxLpwRg15RGQbcK6IWNbRBixYR3BDCgTBOodtyKgx9S+cUUN3AytxhoH+2vs6I7rFMjXRkDqCg2kIncOBLKWEMY5wmobuBy4APlPVEd55Cn4d3WKZmmhIHcHBNITOYZ/AIaOKWv+AiWnhBIITqnpCRBCR5qr6b+/TwKYBagjz/ZYXrIO4PgU2CUmc0PmeztY/YGJaOIEgX0TaAkuA/xORg4TOGWSioCGlhKiuhthBXDCvAM8JD6gNGTUGwussvt776wwRWQEkAe9HtVTGr6GPBKpKQ+wgLni1wJ8HV5pJVJqEsrIgMxPS0yGt/gdGGVOpKgOBiPyvqv4EQFU/9i0DfhLlshka/kigYAKbghrS7GPFWcXkzchDT/miAHS6I/JNQllZMHIklJRAQgJkZFgwMA1bOE1D/QPfeKegHBqd4pjyfCOBfHcEDW0kEJSt+IEKTUH13UFcnFVMwbwCCl4rQEvVeRzSBa7m0RkympnpBAG323nNzLRAYBq2kIFARKYBvwJaiMgh32KgBCcttakD9TkSKFSK6Moq/tuSb6vQFFRfM4+VCQAl+v20SC5oN6odPWb0iErfQHq6cyfguyNIT4/4IYyJqMomr58FzBKRWao6rQ7LZMqJ9kigYBV+qE7e8svLV/xAxJqCirOKKcos8rfhl/89vn08pYWlQV8PZx+uGAAAxLkTiFYQAOfqPyPD+ghM4xFOZ/E0EWkH9AESA5avjGbBTN0IVeGH6uQtvxzKVvwTkycyMXlijZuCfJV/fPt4tjywxT/EEwE9pd//7mviEZyKPtSrj4DEC53u7FQnQ0XT0iwAmMYjnM7iu3EeKusK5AAX4cwmdkV0i2bqQqgKP1Qnb/nloSr+mjQFlRnfL4J6nMpePd4aXcv+HtZrHQcAYxoje7I4xoWq8EM9BVzZ8toqkxLa5Vz9q2jN7ghcztDQPWN68E2nMxk9MZG+doVuTFBS1QyRIrJGVS8QkRzgQlU9KSI5qppSN0UsKzU1VdeuXVsfh26y6nPe4PL9AIFJ4Ho/25vSwtIa9RGUFpayqX17rnuglQ3jNAYQkXWqmhrsM3uyOEZUVtnXV9qH8tk/kzOSSc5I9lf45ZtxAt+H08Tz11k2jNOYcNT0yeJ/RrVUJqIaYpoHCJ79s/u07hFrx7dhnMaEJ5zO4oHAed633/ieLjaNR0NL8+AT7dnBbBinMeGp7IGyJODvwNnAepxuuIEisgO4zjuhfaVEZDTwRyAOeFlVK0x6LyI/wpnfQIH1qnprDb6HqURDSvMAZfsFKmsKigQbxmlM1UJ2FovIHJyniB9RVY93mQt4Amihqr+odMdOKopNwJVAPrAGuEVVcwPW6QO8DVyhqgdF5AxV3VvZfq2zuGbqs0M4ULB+ARvSaUz01bSzeBQwyBcEwD9t5a+AL8M47jBgi6pu9RZiPnAdkBuwzj3AXFU96N1/pUHA1FxDmAcAgvcLWCAwpn5VNlVliaqeKr/Qu+xkGPvuAuwMeJ/vXRboXJy5kD8Rkc+8TUkViMgkEVkrImv37dsXxqFNQ+XrFyAOmxXMmAaisjuCRBEZjNM3EEiA5mHsu/x2UPahf9/x+wDpOE8urxKRAapaZvJYVX0Rb6K71NTUyh98MA1aUlpS1PsFjDHVU1kg2AM8E+KzgjD2nQ90C3jflYrPH+TjPLFcCmwTkY04gWFNGPs3jVRSWpIFAGMakMqyj46o5b7XAH1EpCewC5gAlB8RtAS4BXhdRDrgNBVtreVxjTHGVENlfQS14u1LmAJ8AHwDvK2qX4vI4yIyzrvaB0ChiOQCK4BfqmphtMpk6kdxVjHbZ22nOKu4votijAmiylxDDY0NH21cbLioMQ1DZcNHo3ZHYAwEHy5qjGlYqgwE4vgPEZnufX+2iAyLftFMU2DDRY1p+MLJPvo8Tub3K4DHgcPAIpw5CoyplA0XNabhCycQXKiqQ0QkG8CbCiIhyuUyTYgNFzWmYQunj6DUmzdIAUSkI84dgjHGmCYgnEAwB1gMnCEivwVWA7+LaqlMtWTtzGLWqllk7cyq76IYYxqhcCameVNE1gEjcdJGjFfVb6JeMhOWhjrpjDGm8Qh5RyAip/t+gL3AW8Bfge+8y0wDEGzSGWOMqY7K7gjW4fQLhEoed05USmSqpSFPOmMdxMY0DpXlGupZlwUxNZPWLY2MiRk26YwxpsbCGT6KNzfQcO/bTFV9L3pFMtVlk84YY2ojnCeLnwDux5lZLBe4X0RmRbtgpvGxp4iNaZzCuSMYA6QEzFv8BpANTItmwUzjY08RG9M4hdU0BLQFDnh/t3/dJiR7itiYxiecQDALyBaRFTgjiIZjdwPGGNNkhPNA2VsikomTZE6AqaoazlSVxhhjGoFw5yNwAfuBg8C5IjK8ivWNMcY0ElXeEYjIk8DNwNd8n2xOgZVRLJcxxpg6Ek4fwXigr6qejHZhjDHG1L1wmoa2AvHRLogxxpj6EfKOQET+B6cJ6BiQIyIZgP+uQFXvi37xjDHGRFtlTUNrva/rgKU12bmIjAb+CMQBL6vqE+U+vx34PbDLu+g5VX25JscydccSyxnTtFQWCIqBT1V1b0127J3VbC5wJZAPrBGRpaqaW27VBao6pSbHMHXPEssZ0/RU1kfwHzgPkm0WkddFZJKI9K/GvocBW1R1q6qWAPOB62pT2MYsq7iYWdu3k1VcXN9FqZVgieWMMY1bZWmobwIQkR7Axd6fn4rI2cAaVR1Txb67ADsD3ucDFwZZ70bvcwmbgAdVdWf5FURkEjAJ4Oyzz67isA1PVnExI9evp8TjIcHlIiM5mbSkxnkV7Uss57sjsMRyxjR+VY4aUtU84AucRHM5OLOVtQhj36EmtAn0LtBDVQcBHwJvhCjDi6qaqqqpHTt2DOPQDUtmURElHg9uoMTjIbOo8V5F+xLL9ZzZ05qFjGkiKhs19CsgDegIbAQ+A54DJqmqO4x95wPdAt53BXYHrqCqhQFvXwKeDK/YjUt627YkuFz+O4L0to37KtoSyxnTtFTWWTwROAK8B3wK/EtVq9PAvQboIyI9cUYFTQBuDVxBRDqr6h7v23HAN9XYf6ORlpRERnIymUVFpLdt22ibhYwxTVNlfQTneSepvxhIBx4VkVbAepzRRK9VtmNVPSUiU4APcIaPvqqqX4vI48BaVV0K3Oed/ewUTprr2yPwnRqktKQkCwDGmAZJVMs32wdZSaQZMBQnBfVPgZ6qGhflsgWVmpqqa9eurXpFY4wxfiKyTlVTg31WWR/BOJy7gUuA/jhJ5z4FHva+GmOMaQIq6yO4HafCfwRY530WwBhjTBNTWSC4UatoNxIRqWodY4wxDVtlzxGsEJFfeB8g8xORBBG5wjuJ/W3RLZ4xxphoq+yOYDRwJ/CWdwhoEZCIMwJoOTBbVXOiX0RTXyy5nDGxobLhoyeA54HnRSQe6AAcV9XG+1isCZsllzMmdoQ1Z7GqlqrqHgsCdSdrZxazVs0ia2dWvRzfkssZEzvCmarS1LGsnVmMnDeSEncJCXEJZEzMIK1bWp2WwZLLGRM7LBA0QJl5mZS4S3CrmxJ3CZl5mXUeCHzJ5ayPwJimr8pA4E0T8aaqHqyD8hggvUc6CXEJ/juC9B7p9VIOSy5nTGwI546gE87sYl8ArwIf2LMD0ZXWLY2MiRlk5mWS3iO9zu8G6kJWFmRmQno6pDW9r2dMoxJuriEBrgLuAFKBt4FXVPXb6BavIss11PhlZcHIkVBSAgkJkJFhwcCYaKss11C4o4YUKDJUED8AACAASURBVPD+nALaAQtF5KmIldLEjMxMJwi43c5rZmZ9l8iY2BZOH8F9OE8Q7wdeBn6pqqUi4gI24+QiMiZs6enOnYDvjiA9vb5LZExsC6ePoANwg6puD1yoqh4R+UF0imWasrQ0pznI+giMaRjCCQTLcCaNAUBEWgP9VPVfqtokZxQz0ZeWZgHAmIYinD6CF3CmrPQ56l1mjDGmCQgnEJRJNa2qHuxBtCajOKuY7bO2U5xVnemojTFNSTgV+lZvh7HvLuBeYGv0imTqiiWWM8ZAeHcEP8OZsnIXkA9cCEyKZqFM3bDEcsYYCCMQqOpeVZ2gqmeo6pmqequq7g1n5yIyWkQ2isgWEXm0kvVuEhEVkaAPO5jo8CWWIw5LLGdMDAvnOYJE4C6cCewTfctV9c4qtosD5gJX4txJrBGRpaqaW2691sB9wL+qXXpTK5ZYzhgD4TUN/S9OvqGrgY+BrsDhMLYbBmxR1a3eie/nA9cFWW8m8BRwIqwSm4hKSkui+7TuFgSMiWHhBILeqvoYcFRV3wDGAgPD2K4LsDPgfb53mZ+IDAa6qep7YZbXGGNMhIUTCEq9r0UiMgBIAnqEsZ0EWeYfhupNUTEbeLjKHYlMEpG1IrJ23759YRzaGGNMuMIJBC+KSDvgv4ClQC7wZBjb5QPdAt53BXYHvG8NDAAyRSQPuAhYGqzDWFVfVNVUVU3t2LFjGIc2xhgTrko7i71X7Ye8k9KsBM6pxr7XAH1EpCfO0NMJwK2+D1W1GCePke9YmcD/U1XLMW2MMXWo0jsC71PEU2qyY1U95d32A+Ab4G1V/VpEHheRcTXZpzHGmMgL58ni/xOR/wcswMkzBICqHgi9iX+dZThJ6wKXTQ+xbnoYZTHGGBNh4QQC3/MCkwOWKdVrJoo5WTuz6n2qyeKsYntGoAEqLS0lPz+fEydsxLSJvMTERLp27Up8fHzY21QZCFS1Z61KFYOydmYxct5I/+TzGRMz6jwYWB6hhis/P5/WrVvTo0cPnFlgjYkMVaWwsJD8/Hx69gy/6g7nyeKJIQ44rxrliymZeZmUuEtwq5sSdwmZeZl1HgiC5RGyQNAwnDhxwoKAiQoRoX379lR3mH04TUMXBPyeCIwEvgAsEISQ3iOdhLgE/x1Beo/0Oi+DL4+Q747A8gg1LBYETLTU5G8rnKahX5Q7SBJO2gkTQlq3NDImZtRrH4HlETLGhCucB8rKOwb0iXRBmpq0bmlMu2xavXUUg+URMqF999133HrrrZxzzjkMHTqUtLQ0Fi9eXKt9zpgxgz/84Q8ATJ8+nQ8//LBG+8nJyWHZsmVBP8vMzCQpKYnBgwfTt29fhg8fznvvNawMNbfffjsLFy4M+tkDDzzAypUrAdi2bRsXXnghffr04eabb6akpKTC+oWFhYwYMYJWrVoxZUrZkfyjRo3i4MGDESlzlYFARN4VkaXen/eAjcDfI3J0Y0ydU1XGjx/P8OHD2bp1K+vWrWP+/Pnk5+dXWPfUqVM1Osbjjz/OqFGjarRtZYEA4LLLLiM7O5uNGzcyZ84cpkyZQkZGRo2OVZcOHDjAZ599xvDhwwGYOnUqDz74IJs3b6Zdu3a88sorFbZJTExk5syZ/gAb6Cc/+QnPP/98RMoWzh3BH4CnvT+zgOGqGnJuAWNM5GXtzGLWqllk7cyq9b4++ugjEhIS+NnPfuZf1r17d37xC6cV+PXXX+eHP/wh1157LVdddRVHjhxh5MiRDBkyhIEDB/L3v39/Hfjb3/6Wvn37MmrUKDZu3OhfHnhVvG7dOi6//HKGDh3K1VdfzZ49ewBIT09n6tSpDBs2jHPPPZdVq1ZRUlLC9OnTWbBgASkpKSxYsKDS75KSksL06dN57rnnANi3bx833ngjF1xwARdccAGffPIJAB9//DEpKSmkpKQwePBgDh92Eig/9dRTDBw4kOTkZB591KnWvv32W0aPHs3QoUO57LLL+Pe//+3/Tvfddx8XX3wx55xzjv/7qSpTpkyhX79+jB07lr17g0/XsnDhQkaPHu3f5qOPPuKmm24C4LbbbmPJkiUVtmnZsiWXXnopiYmJFT4bN24cb731VqXnJ2yqWukP0BNIDHjfAuhR1XbR+hk6dKhGyqdFRfq7vDz9tKgoYvs0piq5ubnVWv/THZ9qi9+00Lhfx2mL37TQT3d8Wqvj//GPf9QHHngg5OevvfaadunSRQsLC1VVtbS0VIuLi1VVdd++fdqrVy/1eDy6du1aHTBggB49elSLi4u1V69e+vvf/15VVW+77TZ95513tKSkRNPS0nTv3r2qqjp//ny94447VFX18ssv14ceekhVVf/xj3/oyJEj/cefPHly0LKtWLFCx44dW2ZZdna2nnfeeaqqesstt+iqVatUVXX79u3+5T/4wQ909erVqqp6+PBhLS0t1WXLlmlaWpoePXpUVdX/fa+44grdtGmTqqp+9tlnOmLECP93uummm9TtduvXX3+tvXr1UlXVRYsW6ahRo/TUqVO6a9cuTUpK0nfeeadC2SdOnKhLly4tcx59duzYof3796/0/0mwc9K7d2/dv39/heXB/saAtRqiXg1n1NA7OFNV+ri9yy4IvnrjkFVczMj16ynxeEhwuchITiYtydrSTcMT7eHIkydPZvXq1SQkJLBmzRoArrzySk4//XTAuVj81a9+xcqVK3G5XOzatYvvvvuOVatWcf3113PaaacBzhVqeRs3buSrr77iyiuvBMDtdtO5c2f/5zfccAMAQ4cOJS8vr0bld+o4x4cffkhu7vdzXx06dIjDhw9zySWX8NBDD/HjH/+YG264ga5du/Lhhx9yxx13+Mt/+umnc+TIET799FN++MMf+vdx8uRJ/+/jx4/H5XLRr18/vvvuOwBWrlzJLbfcQlxcHGeddRZXXHFF0HLu2bMHX9LMwDL71GS0zxlnnMHu3btp3759tbcNFE4gaKbOxDIAqGqJiCTU6qgNQGZRESUeD26gxOMhs6jIAoFpkCI9HLl///4sWrTI/37u3Lns37+f1NTvE/+2bNnS//ubb77Jvn37WLduHfHx8fTo0cP/VHRVlZeq0r9/f7KygjdpNW/eHIC4uLga90dkZ2dz/vnnA+DxeMjKyqJFixZl1nn00UcZO3Ysy5Yt46KLLuLDDz9EVSuU3+Px0LZtW3Jyciotr++7+YRTibdo0cJ/3jp06EBRURGnTp2iWbNm5Ofnc9ZZZ4X3hQOcOHGiwnetiXD6CPYFJokTkeuA/bU+cj1Lb9uWBJeLOCDB5SK9rY2zNw2TbzjyzBEzI/KU+hVXXMGJEyd44YUX/MuOHTsWcv3i4mLOOOMM4uPjWbFiBdu3bwdg+PDhLF68mOPHj3P48GHefffdCtv27duXffv2+QNBaWkpX3/9daXla926tb8NvyobNmxg5syZTJ7sZMC56qqr/P0FgL9C//bbbxk4cCBTp04lNTWVf//731x11VW8+uqr/u9+4MAB2rRpQ8+ePXnnnXcAp7Jfv359pWUYPnw48+fPx+12s2fPHlasWBF0vfPPP58tW7YATuAYMWKEv5/hjTfe4Lrrgk3gGJqqUlBQQI8ePaq1XTDhBIKfAb8SkR0isgOYCvy01keuZ2lJSWQkJzOzZ09rFgKysmDWLOfVNDyRHI4sIixZsoSPP/6Ynj17MmzYMG677TaefDL4NCM//vGPWbt2Lampqbz55pucd955AAwZMoSbb76ZlJQUbrzxRi677LIK2yYkJLBw4UKmTp1KcnIyKSkpfPrpp5WWb8SIEeTm5obsLF61apV/+OjkyZOZM2cOI0eOBGDOnDmsXbuWQYMG0a9fP/70pz8B8OyzzzJgwACSk5Np0aIF11xzDaNHj2bcuHGkpqaSkpLiH5nz5ptv8sorr5CcnEz//v3LdI4Hc/3119OnTx8GDhzIz3/+cy6//PKg640dO5bMzEz/+yeffJJnnnmG3r17U1hYyF133QXA0qVLmT79+9ycPXr04KGHHuL111+na9eu/qavdevWcdFFF9GsWTgNO5WTYG1VQVcUaeVdP7xQHSWpqam6dq1NWRBJWVkwciSUlEBCAmRkQFr9Pf7Q5H3zzTf+pgwTWy699FLee+892kagBeL+++9n3Lhx/iAYKNjfmIisU9UKE39BeM8R/E5E2qrqEVU9LCLtROQ3NS28aXgyM50g4HY7rwEXLcaYCHr66afZsWNHRPY1YMCAoEGgJsJpGrpGVYt8b9SZrWxMRI5uGoT0dOdOIC7OeU1Pr+8SGdM0XXjhhQwaNCgi+7rnnnsish8Ib9RQnIg0V9WTACLSAmhexTYmCqI1v0BamtMclJnpBAFrFjImtoQTCP4CZIjIazgT0tyJZR6tc9GeXyAtzQKAMbEqnOyjT4nIBmAUIMBMVf0g6iUzZdj8AsaYaAkr+6iq/lNV/5+qPgwcEZG5US6XKcc3vwBx2PwCxpiICisQiEiKiDwpInnAb4B/R7VUpgLf/AI9Z/a0aSdNrYkIDz/8sP/9H/7wB2bMmFHt/eTl5fHXv/41giULz/Hjx7n88stxu92A80BWnz596NOnD2+88UbQbd555x369++Py+UicAj6l19+ye23314XxW6wQgYCETlXRKaLyDfAc0A+znMEI1T1f8LZuYiMFpGNIrJFRCpkLBWRn4nIlyKSIyKrRaRfjb9JDLD5BUykNG/enL/97W/s31+7JAE1CQS+yrs2Xn31VW644Qbi4uI4cOAAv/71r/nXv/7F559/zq9//eugefoHDBjA3/72N38aaJ+BAweSn58fsWGdjVFldwT/xpmW8lpVvdRb+Yf9f1BE4oC5wDVAP+CWIBX9X1V1oKqmAE8Bz1Sr9MbEiKziYmZt305WcXFE9tesWTMmTZrE7NmzK3xWnVTOjz76KKtWrSIlJYXZs2fjdrv55S9/yQUXXMCgQYP485//DDgTyowYMYJbb72VgQMHAvDMM88wYMAABgwYwLPPPgs4OfoDc+zPmDGDp59+ukIZ33zzTX9Khg8++MCfJK9du3ZceeWV/POf/6ywzfnnn0/fvn2Dno9rr72W+fPnV+cUNimVBYIbgQJghYi8JCIjcTqLwzUM2KKqW71J6+YDZZJpqOqhgLctcUYlRUUk87kbU5d8mXIf27aNkevXRywYTJ48mTfffJPicvu7//77efDBB1mzZg2LFi3i7rvvBpzmo7lz55KTk8OqVato0aIFTzzxBJdddhk5OTk8+OCDvPLKKyQlJbFmzRrWrFnDSy+9xLZt2wD4/PPP+e1vf0tubi7r1q3jtdde41//+hefffYZL730EtnZ2UyYMKFMWom33367TCZQgJKSErZu3erPsbNr1y66devm/7xr167s2rWrWuciNTWVVatWVWubpiTkqCFVXQwsFpGWwHjgQeBMEXkBWKyqy6vYdxdgZ8D7fODC8iuJyGTgISABCJq/VUQmAZMAzj777CoOW1HWzixGzhvpz94YicRdxtSVaGXKbdOmDRMnTmTOnDllMlhWJ5VzecuXL2fDhg3+ZGrFxcVs3ryZhIQEhg0bRs+ePQFYvXo1119/vT/L6Q033MCqVau477772Lt3L7t372bfvn20a9euwr/5/fv3l0nREImUzr50zrGqys5iVT2qqm+q6g+ArkAOEM4MZcH+T1T4P6aqc1W1F04yu/8KUYYXVTVVVVN9+byrI1g+d2Mai2hmyn3ggQd45ZVXOHr0qH+ZL5VzTk4OOTk57Nq1i9atW/Poo4/y8ssvc/z4cS666CL/zF2BVJX/+Z//8W+7bds2rrrqKqBsauvKcpzddNNNLFy4kAULFjBhwoQKnwemcwbnDmDnzu+vOWuS0jlS6Zwbq2pNXq+qB1T1z6oafOaFsvKBbgHvuwKVhdz5OHceEefL5x4ncRHJ525MXYpmptzTTz+dH/3oR2Xmy61OKufyKaOvvvpqXnjhBUpLSwHYtGlTmSDjM3z4cJYsWcKxY8c4evQoixcv9mcvnTBhAvPnz2fhwoX+qRwDtWvXDrfb7Q8GV199NcuXL+fgwYMcPHiQ5cuXc/XVV1frPGzatIkBAwZUa5umpFqBoJrWAH1EpKd3IpsJwNLAFUSkT8DbscDmaBQk0vncjalraUlJTOvePSrp0h9++OEyo4eqk8p50KBBNGvWjOTkZGbPns3dd99Nv379GDJkCAMGDOCnP/1p0AlnhgwZwu23386wYcO48MILufvuuxk8eDDgTJxz+PBhunTpUmY2s0BXXXUVq1evBpxg9thjj/k7t6dPn+6fXe3uu+/2DxVdvHgxXbt2JSsri7Fjx5YJFitWrGDs2LEROJuNU9hpqGu0c5ExwLNAHPCqqv5WRB7HmTtzqYj8EeeJ5VLgIDBFVSudtcLSUJvGztJQ1152djbPPPMM//u//1vrfZ08eZLLL7+c1atXRyS3f0NQ3TTUUf3WqroMWFZu2fSA3++P5vGNMU3T4MGDGTFiBG63m7i4uFrta8eOHTzxxBNNJgjUROx+c2NMo3bnnXdGZD++J5JjWTT7CIwxxjQCFgiMMSbGWSAwxpgYZ4HAGGNinAWCOlScVcz2WdspzopMrhhjamPx4sWISNAnhH1uv/12f7qIcOTl5VX5YFbgOjk5OSxbtqzS9YPZs2cPP/jBD/zvZ82aRe/evenbty8ffBB83qznnnuO3r17IyJlnpt47733+O///u9ql6EpsUBQR3xTTW57bBvrR663YGDq3VtvvcWll15ar1k3axoInnnmGf/k7bm5ucyfP5+vv/6af/7zn9x7771BU11fcsklfPjhh3Tv3r3M8rFjx7J06VKOHTtWsy/RBFggqCPBppo0JlyRvps8cuQIn3zyCa+88kqZQKCqTJkyhX79+jF27Fj27t3r/+zxxx/nggsuYMCAAUyaNMmfL2jdunUkJyeTlpbG3LnfT14YKiW1T0lJCdOnT2fBggWkpKSwYMECPv/8cy6++GIGDx7MxRdfzMaNG4OWf9GiRYwePRqAv//970yYMIHmzZvTs2dPevfuzeeff15hm8GDB/szlgYSEdLT03nvvffCP4FNjAWCOmJTTZqaisbd5JIlSxg9ejTnnnsup59+Ol988QXgNBdt3LiRL7/8kpdeeolPP/3Uv82UKVNYs2YNX331FcePH/dXnHfccQdz5swhK6tsivfKUlIDJCQk8Pjjj3PzzTeTk5PDzTffzHnnncfKlSvJzs7m8ccf51e/+lWFsm/bto127drRvHlzwNJQR4I9UFZHfFNNFmUW0Ta9rc0yZsIW7G6ytn8/b731Fg888ADgJHl76623GDJkCCtXruSWW24hLi6Os846iyuu+D6/5IoVK3jqqac4duwYBw4coH///gwfPpyioiIuv/xyAH7yk5/w/vvvA6FTUp977rkhy1VcXMxtt93G5s2bERF/8rpAe/bsITALsaWhrj0LBHUoKS3JAoCpNt/dpKfEE5G7ycLCQj766CO++uorRAS3242I8NRTTwHBK9ETJ05w7733snbtWrp168aMGTM4ceIEqhqy0vWlpC6fCTQvLy9k2R577DFGjBjB4sWLycvLIz09vcI6loY68qxpyJgGznc32XNmT5Izkmt9MbFw4UImTpzI9u3bycvLY+fOnfTs2ZPVq1czfPhw5s+fj9vtZs+ePaxYsQLAX/F26NCBI0eO+K/y27ZtS1JSkj8T6Jtvvuk/TjgpqcunsS4uLqZLly4AvP7660HLf+6555YJJuPGjWP+/PmcPHmSbdu2sXnzZoYNG1atc2JpqI0xDV5SWhLdp3WPyB3lW2+9xfXXX19m2Y033shf//pXrr/+evr06cPAgQP5+c9/7m/yadu2Lffccw8DBw5k/PjxXHDBBf5tX3vtNSZPnkxaWlqZq+pwUlKPGDGC3Nxcf2fxI488wrRp07jkkktCTnLfsmVLevXqxZYtWwAnbfWPfvQj+vXrx+jRo5k7d64/Ed2YMWP8TT5z5syha9eu5OfnM2jQIP8UnGBpqKOahjoaLA21aewsDXXtLV68mHXr1vGb3/ym1vv67rvvuPXWW8nIyIhAyRqGBpWG2hhjouH666+nsLAwIvvasWMHTz/9dET21VhZIDDGNEqBTTu1EdjMFausj8AYY2KcBQJjjIlxFgiMMSbGWSAwxpgYZ4HAmBiUn5/PddddR58+fejVqxf3338/JSUlAGRmZpKUlMTgwYPp27cvw4cPL5OQbcaMGXTp0oWUlBT/T1FR2SSKHo+H++67jwEDBjBw4EAuuOCCMnmGgunRo0eZ9NCm7kQ1EIjIaBHZKCJbROTRIJ8/JCK5IrJBRDJEpHuw/RhjIkdVueGGGxg/fjybN29m06ZNHDlyhP/8z//0r3PZZZeRnZ3Nxo0bmTNnDlOmTCkzzv7BBx8kJyfH/9O2bdm0FwsWLGD37t1s2LCBL7/8ksWLF1dYJ1rfzePxRP04TU3UAoGIxAFzgWuAfsAtItKv3GrZQKqqDgIWAk9FqzzGNGZZWTBrlvNaWx999BGJiYnccccdAMTFxTF79mxeffXVoDn5U1JSmD59Os8991zYx9izZw+dO3fG5XKqmK5du9KuXTvAebJ54MCBDBgwgKlTp1bYdurUqTz//PP+9zNmzPCP8//973/vT2vtm0wmLy+P888/n3vvvZchQ4aUyTtkwhPNO4JhwBZV3aqqJcB84LrAFVR1har6/vI+A7pGsTzGNEpZWTByJDz2mPNa22Dw9ddfM3To0DLL2rRpw9lnn+1P21DekCFDysxkNnv2bH+z0IgRIyqs/6Mf/Yh3332XlJQUHn74YbKzswHYvXs3U6dO5aOPPiInJ4c1a9awZMmSMttOmDCBBQsW+N+//fbb/PCHP2T58uVs3ryZzz//nJycHNatW8fKlSsB2LhxIxMnTiQ7O7vCxDOmatEMBF2AwNCc710Wyl3A+8E+EJFJIrJWRNbu27cvgkU0puHLzISSEnC7ndfMzNrtL1TG0KoyiQYKbBryJaYL1LVrVzZu3MisWbNwuVyMHDmSjIwM1qxZQ3p6Oh07dqRZs2b8+Mc/9lfmPoMHD2bv3r3s3r2b9evX065dO84++2yWL1/O8uXLGTx4sD8wbd68GYDu3btz0UUX1fSUxLxoPlkc7C8qaGIjEfkPIBW4PNjnqvoi8CI4uYYiVUBjGoP0dEhIcIJAQoLzvjb69+/PokWLyiw7dOgQO3fupFevXkFTN2RnZ1c7P1Lz5s255ppruOaaazjzzDNZsmQJI0eODGvbm266iYULF1JQUMCECRMAJxhNmzaNn/70p2XWzcvLo2XLltUqmykrmncE+UC3gPddgQozP4jIKOA/gXGqejKK5QnJJpU3DVlaGmRkwMyZzmtaWu32N3LkSI4dO8a8efMAZ0rJhx9+mNtvv53TTjutwvobNmxg5syZTJ48OexjfPHFF/6snx6Phw0bNtC9e3cuvPBCPv74Y/bv34/b7eatt97yZzgNNGHCBObPn8/ChQu56aabACet9auvvsqRI0cAZ2aywKk0Tc1F845gDdBHRHoCu4AJwK2BK4jIYODPwGhVrZf/o75pAH2TfkQi37sxkZaWVvsA4CMiLF68mHvvvZeZM2fi8XgYM2YMv/vd7/zrrFq1isGDB3Ps2DHOOOMM5syZU+Zqfvbs2fzlL3/xv1+yZEmZ+YD37t3LPffcw8mTzrXdsGHDmDJlComJicyaNYsRI0agqowZM4brrivTdQg4dy2HDx+mS5cudO7cGYCrrrqKb775hjTviWjVqhV/+ctf/CmnTc1FNQ21iIwBngXigFdV9bci8jiwVlWXisiHwEBgj3eTHao6rrJ9RjoN9fZZ29n22DZwO6XsObMn3adZZ5OJHktDbaKtQaWhVtVlwLJyy6YH/D4qmscPR6SnATTGmMYm5tNQ26TyxphYF/OBAGxSeVP3KhuqaUxt1KS533INGVPHEhMTKSwsrNE/WGMqo6oUFhaSmJhYre3sjsCYOuabQN0ejjTRkJiYSNeu1UvSYIHAmDoWHx9Pz54967sYxvhZ05AxxsQ4CwTGGBPjLBAYY0yMi+qTxdEgIvuA7QGLkoBwkwSFs25V64T6PNjycJZ1AOpqWqbqnKtIbF/Tc1nV53aua7Z+ZevYuY7s+g3xXHdX1Y5BP1HVRv0DvBjJdataJ9TnwZaHswwn3UaDO1eR2L6m59LOdeTPdVXr2LmO7XPdFJqG3o3wulWtE+rzYMvDXVZXanvs6m5f03NZ1ed2rmu2fmXr2LmO7PqN6lw3uqahpkZE1mqIRFAmsuxc1x0713UnEue6KdwRNHYv1ncBYoid67pj57ru1Ppc2x2BMcbEOLsjMMaYGGeBwBhjYpwFAmOMiXEWCBoQEWkpIm+IyEsi8uP6Lk9TJyLniMgrIrKwvsvS1InIeO/f9d9F5Kr6Lk9TJiLni8ifRGShiPw8nG0sEESZiLwqIntF5Ktyy0eLyEYR2SIij3oX3wAsVNV7gErnbjbBVed8q+pWVb2rfkra+FXzXC/x/l3fDtxcD8Vt1Kp5rr9R1Z8BPwLCGlZqgSD6XgdGBy4QkThgLnAN0A+4RUT6AV2Bnd7V3HVYxqbkdcI/36Z2Xqf65/q/vJ+b6nmdapxrERkHrAYywtm5BYIoU9WVwIFyi4cBW7xXpCXAfOA6IB8nGID9v6mRap5vUwvVOdfieBJ4X1W/qOuyNnbV/btW1aWqejEQVhOzVTb1owvfX/mDEwC6AH8DbhSRF6jfR/abmqDnW0Tai8ifgMEiMq1+itbkhPrb/gUwCrhJRH5WHwVrgkL9XaeLyBwR+TOwLJwd2Qxl9SPYrOWqqkeBO+q6MDEg1PkuBKxSiqxQ53oOMKeuC9PEhTrXmUBmdXZkdwT1Ix/oFvC+K7C7nsoSC+x81x0713UnYufaAkH9WAP0EZGeIpIATACW1QtaegAAAilJREFU1nOZmjI733XHznXdidi5tkAQZSLyFpAF9BWRfBG5S1VPAVOAD4BvgLdV9ev6LGdTYee77ti5rjvRPteWdM4YY2Kc3REYY0yMs0BgjDExzgKBMcbEOAsExhgT4ywQGGNMjLNAYIwxMc4CgTHGxDgLBCZmeZPO5Xh/CkRkV8D7T6NwvNtFZJ+IvFzJOi28xy8RkQ6RLoMxwVjSOROzvEnnUgBEZAZwRFX/EOXDLlDVKZWU6TiQIiJ5US6HMX52R2BMECJyxPuaLiIfi8jbIrJJRJ4QkR+LyOci8qWI9PKu11FEFonIGu/PJWEco793PzkiskFE+kT7exkTjN0RGFO1ZOB8nIlBtgIvq+owEbkfJ8/+A8AfgdmqulpEzsbJ/3J+Ffv9GfBHVX3TmzQsLmrfwJhKWCAwpmprVHUPgIh8Cyz3Lv8SGOH9fRTQT8SfIr6NiLRW1cOV7DcL+E8R6Qr8TVU3R77oxlTNmoaMqdrJgN89Ae89fH8x5QLSVDXF+9OliiCAqv4VGAccBz4QkSsiXG5jwmKBwJjIWI6TEhgAEUmpagMROQfY6p29aykwKHrFMyY0CwTGRMZ9QKq30zeX8KbAvBn4SkRygPOAedEsoDGh2HwExtQREbkdSK1s+GjAunnedfdHu1zG2B2BMXXnOHBNOA+UAfE4fRDGRJ3dERhjTIyzOwJjjIlxFgiMMSbGWSAwxpgYZ4HAGGNinAUCY4yJcf8fBtzB5eQP/c8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(train_times[0], all_accuracy[0], '.', color='g', label='Gradient Descend (0.1)')\n",
    "plt.plot(train_times[1], all_accuracy[1], '.', color='c', label='Nesterov (0.1)')\n",
    "plt.plot(train_times[2], all_accuracy[2], '.', color='m', label='Adadelta (0.1)')\n",
    "plt.plot(times_ode, all_accuracy_results_ode, '.', color='b',label='ODE Solver')#o-\n",
    "plt.semilogx()\n",
    "plt.xlabel('Time [s]')\n",
    "plt.ylabel('Accuracy (Whole Dataset)')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 84.386%\n",
      "1 85.077%\n",
      "2 83.786%\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(weights_save)):\n",
    "    model.set_weights(weights_save[i])\n",
    "    accuracy_all = tfe.metrics.Accuracy()\n",
    "    accuracy_all(tf.argmax(model(features_all), axis=1, output_type=tf.int32),\n",
    "                 tf.argmax(labels_all, axis=1, output_type=tf.int32))\n",
    "    print(i, '{:.3%}'.format(accuracy_all.result()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "84.531%\n"
     ]
    }
   ],
   "source": [
    "model.set_weights(best_weights_ode)\n",
    "accuracy_all = tfe.metrics.Accuracy()\n",
    "accuracy_all(tf.argmax(model(features_all), axis=1, output_type=tf.int32),\n",
    "             tf.argmax(labels_all, axis=1, output_type=tf.int32))\n",
    "print('{:.3%}'.format(accuracy_all.result()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}