Created
September 5, 2021 23:50
-
-
Save paloha/b3262376debaf18fd33b5961b3a51ebf to your computer and use it in GitHub Desktop.
stackoverflow-69055300-answer.ipynb
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "nbformat": 4, | |
| "nbformat_minor": 0, | |
| "metadata": { | |
| "colab": { | |
| "name": "stackoverflow-69055300-answer.ipynb", | |
| "provenance": [], | |
| "collapsed_sections": [], | |
| "toc_visible": true, | |
| "authorship_tag": "ABX9TyNCiNKm9L+HALXSNhMfuR4a", | |
| "include_colab_link": true | |
| }, | |
| "kernelspec": { | |
| "name": "python3", | |
| "display_name": "Python 3" | |
| }, | |
| "language_info": { | |
| "name": "python" | |
| } | |
| }, | |
| "cells": [ | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "view-in-github", | |
| "colab_type": "text" | |
| }, | |
| "source": [ | |
| "<a href=\"https://colab.research.google.com/gist/paloha/b3262376debaf18fd33b5961b3a51ebf/stackoverflow-69055300-answer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "dOhKlOsLXAbJ" | |
| }, | |
| "source": [ | |
| "Code supporting my answer to the stackoverflow question no. 69055300.\n", | |
| "\n", | |
| "https://stackoverflow.com/questions/69055300" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "KFqvpUYyFDqf" | |
| }, | |
| "source": [ | |
| "import os\n", | |
| "import numpy as np\n", | |
| "import pandas as pd\n", | |
| "import matplotlib.pyplot as plt\n", | |
| "from imageio import imread, imwrite\n", | |
| "from tensorflow.keras.datasets import mnist\n", | |
| "from tensorflow.keras.preprocessing.image import ImageDataGenerator" | |
| ], | |
| "execution_count": 2, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "roJ88oZWMljX" | |
| }, | |
| "source": [ | |
| "# Preparing the data" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/" | |
| }, | |
| "id": "VbVQw06CBtlK", | |
| "outputId": "7dc6c308-2d9b-4b64-d7ff-7c5a546ebb78" | |
| }, | |
| "source": [ | |
| "# Loading mnist test set from tensorflow\n", | |
| "_, (x_test, y_test) = mnist.load_data()\n", | |
| "x_test = np.expand_dims(x_test, -1) # Adding channels axis" | |
| ], | |
| "execution_count": 3, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "name": "stdout", | |
| "text": [ | |
| "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n", | |
| "11493376/11490434 [==============================] - 0s 0us/step\n", | |
| "11501568/11490434 [==============================] - 0s 0us/step\n" | |
| ] | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "Jl-scm8BCN6F" | |
| }, | |
| "source": [ | |
| "# Saving the images on a hard drive\n", | |
| "!mkdir mnist\n", | |
| "!cd mnist && mkdir 0 1 2 3 4 5 6 7 8 9\n", | |
| "\n", | |
| "fpaths = []\n", | |
| "for i, (img, label) in enumerate(zip(x_test, y_test)):\n", | |
| " fpath = os.path.join('mnist', f'{label}', f'{i}.jpg')\n", | |
| " imwrite(fpath, img)\n", | |
| " fpaths.append(fpath)" | |
| ], | |
| "execution_count": 4, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "hFM2qtayUg5Z" | |
| }, | |
| "source": [ | |
| "# Initiating a memmap on drive with specified shape and dtype\n", | |
| "mmap = np.memmap(f'mnist.mmap', dtype='uint8', mode='w+', shape=(x_test.shape))\n", | |
| "\n", | |
| "# Filling the memmap with data\n", | |
| "# If hard disk space is a problem, we can delete the source image files on the go\n", | |
| "for i, fpath in enumerate(fpaths):\n", | |
| " mmap[i][:] = np.expand_dims(imread(fpath)[:], -1)\n", | |
| " # deleting the file if desired\n", | |
| " # os.remove(fpath)" | |
| ], | |
| "execution_count": 5, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "m5xhVNoKMoRn" | |
| }, | |
| "source": [ | |
| "# Comparing types of loading data\n", | |
| "\n", | |
| "We are using `tensorflow.keras.ImageDataGenerator` and comparing how long does it take to generate all the images from the mnist test set (10k) 2 times depending on how we load the data." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "TNioAfJtSWPn" | |
| }, | |
| "source": [ | |
| "batch_size = 8\n", | |
| "n_batches = int(np.ceil(len(x_test) / batch_size))\n", | |
| "epochs = 2" | |
| ], | |
| "execution_count": 58, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "LyytJ1xJOm8a" | |
| }, | |
| "source": [ | |
| "## From numpy array" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "qQGEpF42FMeA" | |
| }, | |
| "source": [ | |
| "DataGen = ImageDataGenerator(rescale=1./255)\n", | |
| "gen = DataGen.flow(x_test, y=None, batch_size=batch_size, shuffle=True, seed=10)" | |
| ], | |
| "execution_count": 59, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/" | |
| }, | |
| "id": "br7ejWQ2FuR6", | |
| "outputId": "8ca2a3a2-8c19-47bd-fa4f-119d2386fc9c" | |
| }, | |
| "source": [ | |
| "%%time\n", | |
| "for i in range(n_batches * epochs):\n", | |
| " n = next(gen)" | |
| ], | |
| "execution_count": 60, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "name": "stdout", | |
| "text": [ | |
| "CPU times: user 287 ms, sys: 1.69 ms, total: 289 ms\n", | |
| "Wall time: 293 ms\n" | |
| ] | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 298 | |
| }, | |
| "id": "7FrHkE6iPgQy", | |
| "outputId": "601510ce-0c16-4014-d34d-10d0e5c384a4" | |
| }, | |
| "source": [ | |
| "plt.title(f'last batch out shape: {n.shape}')\n", | |
| "plt.imshow(n[0].squeeze())" | |
| ], | |
| "execution_count": 61, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "data": { | |
| "text/plain": [ | |
| "<matplotlib.image.AxesImage at 0x7f6fcdef6f50>" | |
| ] | |
| }, | |
| "metadata": {}, | |
| "execution_count": 61 | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAEICAYAAACZA4KlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAASmklEQVR4nO3de7BdZX3G8e8TckEgQIIYDxANcikXpVFPI46oqQELYWqgZaiINjpqsMIIFrUUh4Gpl2GQiLRaaCwYRAggGkGKVghUpColQLgEUC6GJvEkAcIlgRIS8usf6z1k5bBv5+wrvM9nZs9Ze71rv+u319nPXtdzliICM3vtG9XtAsysMxx2s0w47GaZcNjNMuGwm2XCYTfLRNfDLmmZpEM7MJ+QtHcH5jNf0lfbPZ9mSJouaUW366hE0jhJ90vq63YtvUrSXEl/N9zXdT3srdDugHXqC2k4ejmwTZoD3BIRA/By+C+UtFrSWkk/lbR7Ix1JOljSDel1j0v6YflLpIf7Hivp6vS5C0nTh0xyLnC6pLGN9DfoNRF2e035DHBp6fnJwLuBg4DdgKeAf2mwrwnAPGAK8GZgHfC9V0HfALcCHwVWDW1IX4QPAh8aRn8QEV19AMuAQ9PwNOA3wNPAAPBtYGxqE3AesAZ4FrgXeCvFmmAj8CKwHvhplfkE8DngUeAJ4BvAqNS2F3AT8GRquwzYObVdCmwG/i/1/6U0/hDg16nW5cDH0/j5wHeA/6D4ANwG7FXj/X8IWJr6+S9g/yE17116Ph/4KrB9qmdzqmk9sFuFvmcC96c6VgJfSOOnAyuAU9PyHAA+UXrdkcBdaTkvB84qtU1Jdc0B/phe+4VS+yjgNOCRtDyvAiY2+Fl4U3pfo0vjLgDOGVLb70b4WXsHsO7V1Hf6PU2vMP7LwPeG1Vc7AjzMN7OMLWF/J3AwMDp9qB4ATkltfwHcAexMEfz9gb5yCOrMJ4CbgYnpQ/V74FOpbW/gMGAcsCtwC/CtSjWm54Pf5McBY4BdgKmlWp6k+OIaTfHFcUWVmvYFnkvzHgN8CXiYLV9wFcOehqcDK+q85wHgvWl4AvCO0ms3Af+U5jsTeB6YUGp/G0VwDwJWA0eltimprgUUXzpvAx4v/Q5PBn4L7JGW578BC0o13QN8pEq9RwJLh4zrB/6bYu24HXB5+XczzM/aKcBvX019Uz3sfwXcOay+2h3mBt7MVkGqsAAXpuEPUAT0YNIauVIIaswngMNLzz8LLKoy7VHAXdVqBP5xsK4Kr50P/Hvp+UzgwSrTngFcVXo+imINPL1UczNh/1/gBGDHIeOn88o16Brg4Cr9fAs4Lw0Phn2/Uvs5wEVp+AFgRqmtj2LLa3StWtO0x5cDk8btBFyR5rmJYoujoS2FIf0cBKwlffm9ivquFvbDgEeH01dP7bNL2lfSdZJWSXoW+DrweoCIuIlis/47wBpJ8yTtOMxZLC8NP0bxrYukSZKukLQyzfcHg/OtYjLFZmo15f2s54Edqky3W6oDgIjYnGps6EBOA/6a4svmMUm/lPTuUtuTEbGpUp2S3iXp5nTg6RmK/eihy6PisqTY6lko6WlJT1OE/yVgUgP1PgWMHzLuOxRbCLtQbEn8GPhZA329LJ2F+RlwckT86tXSdx3jKXb9GtZTYafYz3kQ2CcidgROp9hkByAi/jki3gkcQLEJ/MXBpgb7n1wafhPFPicUXyoBvC3N96Pl+VbofznFfn6z/kgRDgAkKdW4Mo16nmITcNAba9T0ChFxe0TMAt4A/IRi/7kRlwPXApMjYifgQrZeHlB9WS4HjoiInUuPbSNiJfXdA+wpaXRp3FRgfkSsjYgNFAe5pkmq9WX8MklvBm4EvhIRlw5p7tm+G7A/cPdwXtBrYR9PcVBovaT9gJfPJUr6s7TGGUOxn/sCxQEqKPYp39JA/1+UNEHSZIp9yytL810PPJNOj3xxyOuG9n8ZcKikYyWNlrSLpKnDeqeFq4AjJc1I7+tUYAPFgT+AJcBHJG0j6XDg/UNq2kXSTpU6Tqdvjpe0U0RspFiumytNW8F4YG1EvCBpGvCRCtOcIWk7SQcCn2DLsrwQ+FoKApJ2lTSrkZlGxAqKYxbTSqNvB/5W0k5pGX0W+GNEPJH6ny9pfqX+0u/yJuDbEXFhhUl6su/UPk7StunpWEnbppXBoPcz3C2F4e5DtPrB1gfo3kexZl8P/IriANKtqW0GxTf/erYcMd8hte1DEYyngZ9UmU/5aPyTwFxgm9R2IMXBv/Wpn1Mp7Q8Dsyj2f59myxHt91IcaR88Yj07jZ9P6fgBdfatgaMpjpg/A/wSOLDU1k9xpH4dxVmBBUP6vji9l6cZcjQeGAv8nGLT+FmKD98h1Woa8ns4hmLTfB1wHcXu0w9S2xS2Phq/inSGIrWPAv4e+F16/SPA10vtS4HjayyPE4ELSs93Sb/rNel93gpMK7UvAj5dpa8zU63ry49e77v0+4ghjymprY9iX37scLKm9GKzhkiaAvwBGBNb7/O3qv9xFAezZkS6sKbGtGMpNmUPSlsvrayjZ/uWNBd4JCL+dVivc9htONoddmufXttnN7M28ZrdLBNes5tlYnT9SVpnrMbFtmzfyVmaZeUFnuPF2DD0mgigybCnc7/nA9tQXCJ6dq3pt2V73qUZzczSzGq4LRZVbRvxZrykbSguCTyC4oq24yQdMNL+zKy9mtlnnwY8HBGPRsSLFBf9N3SllJl1XjNh352t/xhiBRX+gEPSHEmLJS3eyIYmZmdmzWj70fiImBcR/RHRP4Zx7Z6dmVXRTNhXsvVfPu3Blr/WMrMe00zYbwf2kbRnutb3wxR/FmlmPWjEp94iYpOkk4D/pDj1dnFELG1ZZWbWUk2dZ4+I64HrW1SLmbWRL5c1y4TDbpYJh90sEw67WSYcdrNMOOxmmXDYzTLhsJtlwmE3y4TDbpYJh90sEw67WSYcdrNMOOxmmXDYzTLhsJtlwmE3y4TDbpYJh90sEw67WSYcdrNMOOxmmXDYzTLhsJtlwmE3y4TDbpYJh90sEw67WSYcdrNMOOxmmWjqls2SlgHrgJeATRHR34qizKz1mgp78ucR8UQL+jGzNvJmvFkmmg17AL+QdIekOZUmkDRH0mJJizeyocnZmdlINbsZf0hErJT0BuAGSQ9GxC3lCSJiHjAPYEdNjCbnZ2Yj1NSaPSJWpp9rgIXAtFYUZWatN+KwS9pe0vjBYeCDwH2tKszMWquZzfhJwEJJg/1cHhE/b0lV1jKbPvDOmu3jz1xRs33S656t2X7v3D+t3f+Vv63Zbp0z4rBHxKNA7d+0mfUMn3ozy4TDbpYJh90sEw67WSYcdrNMtOIPYayHPT51XM326/e+rqn+nz93Uc32WS+cUrXtddf8T1PztuHxmt0sEw67WSYcdrNMOOxmmXDYzTLhsJtlwmE3y4TPs7/G7XbzMzXbrz7hjTXbj9lhVc32HUbVPo+/4+eXV23beE3Nl1qLec1ulgmH3SwTDrtZJhx2s0w47GaZcNjNMuGwm2XC59lf4+KupTXbv/aDv6nZfsxnzm9q/p/b48aqbXM5sKm+bXi8ZjfLhMNulgmH3SwTDrtZJhx2s0w47GaZcNjNMuGwm2WibtglXSxpjaT7SuMmSrpB0kPp54T2lmlmzWpkzT4fOHzIuNOARRGxD7AoPTezHlY37BFxC7B2yOhZwCVp+BLgqBbXZWYtNtJr4ydFxEAaXgVMqjahpDnAHIBt2W6EszOzZjV9gC4iAoga7fMioj8i+sdQ+58Tmln7jDTsqyX1AaSfa1pXkpm1w0jDfi0wOw3PBvxPgc16XCOn3hYAvwH+RNIKSZ8EzgYOk/QQcGh6bmY9rO4Buog4rkrTjBbXYmZt5CvozDLhsJtlwmE3y4TDbpYJh90sEw67WSYcdrNMOOxmmXDYzTLhsJtlwmE3y4TDbpYJh90sEw67WSYcdrNMOOxmmXDYzTLhsJtlwmE3y4TDbpYJh90sEyO9/ZNZQ0764aeqtu3JbzpYiXnNbpYJh90sEw67WSYcdrNMOOxmmXDYzTLhsJtlwufZra1GP6dul2BJI/dnv1jSGkn3lcadJWmlpCXpMbO9ZZpZsxrZjJ8PHF5h/HkRMTU9rm9tWWbWanXDHhG3AGs7UIuZtVEzB+hOknRP2syfUG0iSXMkLZa0eCMbmpidmTVjpGG/ANgLmAoMAHOrTRgR8yKiPyL6xzBuhLMzs2aNKOwRsToiXoqIzcB3gWmtLcvMWm1EYZfUV3p6NHBftWnNrDfUPc8uaQEwHXi9pBXAmcB0SVOBAJYBJ7SxRmujqHMafFSd9cEYbVO7A59m7xl1wx4Rx1UYfVEbajGzNvLlsmaZcNjNMuGwm2XCYTfLhMNulgn/iWvmFLXbN7O5ZvuGqN0++vnhVmTt4jW7WSYcdrNMOOxmmXDYzTLhsJtlwmE3y4TDbpYJn2e3pqzYVPtfjfXN/XWHKrF6vGY3y4TDbpYJh90sEw67WSYcdrNMOOxmmXDYzTLhsJtlwmE3y4TDbpYJh90sEw67WSYcdrNMOOxmmXDYzTLRyC2bJwPfByZR3KJ5XkScL2kicCUwheK2zcdGxFPtK9V60RG/PrFm+57c3aFKrJ5G1uybgFMj4gDgYOBESQcApwGLImIfYFF6bmY9qm7YI2IgIu5Mw+uAB4DdgVnAJWmyS4Cj2lWkmTVvWPvskqYAbwduAyZFxEBqWkWxmW9mParhsEvaAfgRcEpEPFtui4ig2J+v9Lo5khZLWryR2v+vzMzap6GwSxpDEfTLIuLHafRqSX2pvQ9YU+m1ETEvIvojon8M41pRs5mNQN2wSxJwEfBARHyz1HQtMDsNzwauaX15ZtYqjfwr6fcAHwPulbQkjTsdOBu4StIngceAY9tTorXTlAUDNdsf/3SdXa9693y2nlE37BFxK6AqzTNaW46ZtYuvoDPLhMNulgmH3SwTDrtZJhx2s0w47GaZ8C2bM/fSw3+o2b6hzmn0/foqXji55fXDLcjaxmt2s0w47GaZcNjNMuGwm2XCYTfLhMNulgmH3SwTPs9uTTlvytU12z/LIR2qxOrxmt0sEw67WSYcdrNMOOxmmXDYzTLhsJtlwmE3y4TPs1tNZ6z4y5rtX9njpx2qxJrlNbtZJhx2s0w47GaZcNjNMuGwm2XCYTfLhMNulom659klTQa+D0wCApgXEedLOgv4NPB4mvT0iLi+XYVadyw/d9+a7Z8/5Zg6PaxqXTHWlEYuqtkEnBoRd0oaD9wh6YbUdl5EnNu+8sysVeqGPSIGgIE0vE7SA8Du7S7MzFprWPvskqYAbwduS6NOknSPpIslTajymjmSFktavNE3AzLrmobDLmkH4EfAKRHxLHABsBcwlWLNP7fS6yJiXkT0R0T/GMa1oGQzG4mGwi5pDEXQL4uIHwNExOqIeCkiNgPfBaa1r0wza1bdsEsScBHwQER8szS+rzTZ0cB9rS/PzFqlkaPx7wE+BtwraUkadzpwnKSpFKfjlgEntKVC66rtFt5Ws33Dwg4VYk1r5Gj8rYAqNPmcutmriK+gM8uEw26WCYfdLBMOu1kmHHazTDjsZplw2M0y4bCbZcJhN8uEw26WCYfdLBMOu1kmHHazTDjsZplQRHRuZtLjwGOlUa8HnuhYAcPTq7X1al3g2kaqlbW9OSJ2rdTQ0bC/YubS4ojo71oBNfRqbb1aF7i2kepUbd6MN8uEw26WiW6HfV6X519Lr9bWq3WBaxupjtTW1X12M+ucbq/ZzaxDHHazTHQl7JIOl/Q7SQ9LOq0bNVQjaZmkeyUtkbS4y7VcLGmNpPtK4yZKukHSQ+lnxXvsdam2syStTMtuiaSZXaptsqSbJd0vaamkk9P4ri67GnV1ZLl1fJ9d0jbA74HDgBXA7cBxEXF/RwupQtIyoD8iun4BhqT3AeuB70fEW9O4c4C1EXF2+qKcEBH/0CO1nQWs7/ZtvNPdivrKtxkHjgI+TheXXY26jqUDy60ba/ZpwMMR8WhEvAhcAczqQh09LyJuAdYOGT0LuCQNX0LxYem4KrX1hIgYiIg70/A6YPA2411ddjXq6ohuhH13YHnp+Qp6637vAfxC0h2S5nS7mAomRcRAGl4FTOpmMRXUvY13Jw25zXjPLLuR3P68WT5A90qHRMQ7gCOAE9Pmak+KYh+sl86dNnQb706pcJvxl3Vz2Y309ufN6kbYVwKTS8/3SON6QkSsTD/XAAvpvVtRrx68g276uabL9bysl27jXek24/TAsuvm7c+7EfbbgX0k7SlpLPBh4Nou1PEKkrZPB06QtD3wQXrvVtTXArPT8Gzgmi7WspVeuY13tduM0+Vl1/Xbn0dExx/ATIoj8o8AX+5GDVXqegtwd3os7XZtwAKKzbqNFMc2PgnsAiwCHgJuBCb2UG2XAvcC91AEq69LtR1CsYl+D7AkPWZ2e9nVqKsjy82Xy5plwgfozDLhsJtlwmE3y4TDbpYJh90sEw67WSYcdrNM/D+QmlnwF1IFAQAAAABJRU5ErkJggg==\n", | |
| "text/plain": [ | |
| "<Figure size 432x288 with 1 Axes>" | |
| ] | |
| }, | |
| "metadata": { | |
| "needs_background": "light" | |
| } | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "ImUWhtkDOqps" | |
| }, | |
| "source": [ | |
| "## From a directory" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/" | |
| }, | |
| "id": "w0th8N6pGD9m", | |
| "outputId": "d8353ddd-ae42-465f-b6cf-cb8d2013394c" | |
| }, | |
| "source": [ | |
| "DataGen = ImageDataGenerator(rescale=1./255)\n", | |
| "gen = DataGen.flow_from_directory('mnist', batch_size=batch_size, shuffle=True, \n", | |
| " seed=10, class_mode=None, target_size=(28, 28), \n", | |
| " color_mode='grayscale')" | |
| ], | |
| "execution_count": 62, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "name": "stdout", | |
| "text": [ | |
| "Found 10000 images belonging to 10 classes.\n" | |
| ] | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/" | |
| }, | |
| "id": "bQjoZ1hjGOdH", | |
| "outputId": "f09358d3-314d-45a5-86b7-20a8f4115138" | |
| }, | |
| "source": [ | |
| "%%time\n", | |
| "for i in range(n_batches * epochs):\n", | |
| " n = next(gen)" | |
| ], | |
| "execution_count": 63, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "name": "stdout", | |
| "text": [ | |
| "CPU times: user 3.2 s, sys: 286 ms, total: 3.48 s\n", | |
| "Wall time: 3.5 s\n" | |
| ] | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 298 | |
| }, | |
| "id": "v5REInDZPU5s", | |
| "outputId": "453a6b07-776c-4182-bc8f-e5b6bb0ca204" | |
| }, | |
| "source": [ | |
| "# For some reason, even though the seed is set the same\n", | |
| "# the resulting image from this generator is different.\n", | |
| "plt.title(f'last batch out shape: {n.shape}')\n", | |
| "plt.imshow(n[0].squeeze())" | |
| ], | |
| "execution_count": 64, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "data": { | |
| "text/plain": [ | |
| "<matplotlib.image.AxesImage at 0x7f6fcdc3ffd0>" | |
| ] | |
| }, | |
| "metadata": {}, | |
| "execution_count": 64 | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAEICAYAAACZA4KlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAWuUlEQVR4nO3de5RdZXnH8e9vLklMSGISYogYCSpWUChqjFippuIFcS3B1sUSL0WXGlu1FYtai8sltdbFUvFWrTYWDN5QWkVR8YJoRWpBAiJ3L1AoCblBCEkIuczM0z/2DpwM57z7zJzr8P4+a82aM/s5+93P7Jnn7Mu7934VEZjZI99ArxMws+5wsZtlwsVulgkXu1kmXOxmmXCxm2Wi58Uu6XZJL+zCckLSk7qwnNWSPtTp5bRC0gpJa3udRz2Spku6SdLiXufSrySdLemvJzpfz4u9HTpdYN36QJqIfi7YFq0ELouI9fBg8X9e0kZJWyR9V9LBzTQk6RhJl5TzbZb0H7UfIn3c9jRJ/1n+34WkFePe8jHgDEnTmmlvn0dEsdsjyl8BX675+R3Ac4CjgMcC9wL/0mRb84BVwFLgEGA78MUp0DbA5cBrgQ3jA+UH4S3AyyfQHkRET7+A24EXlq+XA/8DbAXWA58BppUxAZ8ANgHbgOuBp1FsCfYCe4AdwHcbLCeAvwVuA+4GPgoMlLEnAj8F7iljXwUeXca+DIwBD5Ttv6ecfizwyzLXO4HXl9NXA58Fvk/xD3Al8MTE7/9y4Maynf8CDh+X85Nqfl4NfAiYVeYzVua0A3hsnbZPAG4q81gHvKucvgJYC5xers/1wBtq5nsZ8OtyPd8JnFkTW1rmtRK4q5z3XTXxAeC9wK3l+rwAmN/k/8Ljy99rqGba54CPjMvtt5P8X3sGsH0qtV3+nVbUmf4+4IsTaqsTBTzBX+Z2Hir2ZwLHAEPlP9XNwGll7CXA1cCjKQr/cGBxbRFULCeAnwHzy3+q3wFvKmNPAl4ETAcWApcBn6yXY/nzvk/yU4BhYAFwdE0u91B8cA1RfHB8vUFOTwbuL5c9DLwH+AMPfcDVLfby9QpgbcXvvB740/L1POAZNfOOAB8sl3sCsBOYVxM/kqJwjwI2AieVsaVlXudTfOgcCWyu+Ru+A7gCeFy5Pv8NOL8mp+uAVzfI92XAjeOmLQP+m2LrOBP4Wu3fZoL/a6cBV0yltmlc7H8OXDOhtjpdzE38MvsVUp0VeGH5+gUUBXoM5Ra5XhEklhPA8TU/vxW4tMF7TwJ+3ShH4B/25VVn3tXAv9f8fAJwS4P3vh+4oObnAYot8IqanFsp9v8D3gLMGTd9BQ/fgm4CjmnQzieBT5Sv9xX7U2riHwHOKV/fDBxXE1tMsec1lMq1fO9ragumnDYX+Hq5zBGKPY6m9hTGtXMUsIXyw28Ktd2o2F8E3DaRtvrqmF3SkyV9T9IGSduADwMHAkTETyl26z8LbJK0StKcCS7izprXd1B86iJpkaSvS1pXLvcr+5bbwBKK3dRGao+zdgIHNHjfY8s8AIiIsTLHpk7kNOEvKD5s7pD0c0nPqYndExEj9fKU9GxJPytPPN1HcRw9fn3UXZcUez0XStoqaStF8Y8Ci5rI915g9rhpn6XYQ1hAsSfxLeAHTbT1oLIX5gfAOyLiF1Ol7QqzKQ79mtZXxU5xnHMLcFhEzAHOoNhlByAiPh0RzwSOoNgFfve+UJPtL6l5/XiKY04oPlQCOLJc7mtrl1un/TspjvNbdRdFcQAgSWWO68pJOyl2Afc5KJHTw0TEVRFxIvAY4NsUx8/N+BpwEbAkIuYCn2f/9QGN1+WdwEsj4tE1XzMiYh3VrgMOlTRUM+1oYHVEbImI3RQnuZZLSn0YP0jSIcBPgH+KiC+PC/dt2004HPjNRGbot2KfTXFSaIekpwAP9iVKela5xRmmOM7dRXGCCopjyic00f67Jc2TtITi2PIbNcvdAdxXdo+8e9x849v/KvBCSSdLGpK0QNLRE/pNCxcAL5N0XPl7nQ7spjjxB3At8GpJg5KOB54/LqcFkubWa7jsvnmNpLkRsZdivY7Ve28ds4EtEbFL0nLg1XXe835JMyU9FXgDD63LzwP/XBYCkhZKOrGZhUbEWopzFstrJl8F/KWkueU6eitwV0TcXba/WtLqeu2Vf8ufAp+JiM/XeUtftl3Gp0uaUf44TdKMcmOwz/OZ6J7CRI8h2v3F/ifonkexZd8B/ILiBNLlZew4ik/+HTx0xvyAMnYYRWFsBb7dYDm1Z+PvAc4GBsvYUylO/u0o2zmdmuNh4ESK49+tPHRG+08pzrTvO2N9ajl9NTXnD6g4tgZeQXHG/D7g58BTa2LLKM7Ub6foFTh/XNvnlr/LVsadjQemAT+k2DXeRvHPd2yjnMb9HV5JsWu+HfgexeHTV8rYUvY/G7+BsoeijA8Afwf8tpz/VuDDNfEbgdck1sfbgM/V/Lyg/FtvKn/Py4HlNfFLgTc3aOsDZa47ar/6ve2av0eM+1paxhZTHMtPm0itqZzZrCmSlgL/CwzH/sf87Wp/OsXJrOOivLAm8d5pFLuyR5V7L+3Mo2/blnQ2cGtE/OuE5nOx20R0utitc/rtmN3MOsRbdrNMeMtulomh6re0zzRNjxnM6uYi+4IG0p+pMdZsj9hkFj6+e3z8wtN7dj3NvUqLv1uy6cHBdNOjo5Nuu5N2cT97YnfdFdNSsZd9v58CBikuET0r9f4ZzOLZOq6VRU5JA4+amYyP7dzZsWVr+vRkPHbvTsYHDhh/Qdv+xrZvn3BO7aLh9B2esXdP4+BAupgH5zS66LEwuvW+ZLxXroxLG8YmvRsvaZDiksCXUlzRdoqkIybbnpl1VivH7MuBP0TEbRGxh+Ki/6aulDKz7mul2A9m/5sh1lLnBg5JKyWtkbRmL+ldRjPrnI6fjY+IVRGxLCKWDZM+fjSzzmml2Nex/51Pj+Ohu7XMrM+0UuxXAYdJOrS81vdVFLdFmlkfmnTXW0SMSHo78COKrrdzI+LGtmX2CFLZtVbRXzxQ0X02tmtXw1hV11qV2JPovmrCwIwZjYPDw+mZK/qyW+qyHEu3HaMV1w90sI+/U1rqZ4+Ii4GL25SLmXWQL5c1y4SL3SwTLnazTLjYzTLhYjfLhIvdLBNdvZ/d6huYmb4Ftqq/uRWt3gJbJXUNAKlYEwZmp2+/1WDjbVnVLaqVt+5W9bP3IW/ZzTLhYjfLhIvdLBMudrNMuNjNMuFiN8uEu966YGBW+vHZVd1bMdK5UZY0lP4XqMqtqutO0xo/Abaye6viCbCxq2K9DSS6x1q8RXXgUY9Kxjv5xODJ8pbdLBMudrNMuNjNMuFiN8uEi90sEy52s0y42M0y4X72Lhi7//6W5q/qyx6c9+jGy96+Izlvy7lV9NO3MsrrwKyK0W9baTv1iGsqbs2lP/vRq3jLbpYJF7tZJlzsZplwsZtlwsVulgkXu1kmXOxmmXA/excMHbQoGR/ZsDEZr7qnvGr+lMEDFyTjv//0kmT8puefk4wftepvGsYO/VR6hO+qxz1X9fErdc/5WMWQzI9ALRW7pNuB7cAoMBIRy9qRlJm1Xzu27H8WEXe3oR0z6yAfs5tlotViD+DHkq6WtLLeGyStlLRG0pq9tDaUkJlNXqu78cdGxDpJjwEukXRLRFxW+4aIWAWsApij+emn+JlZx7S0ZY+IdeX3TcCFwPJ2JGVm7TfpYpc0S9Lsfa+BFwM3tCsxM2uvVnbjFwEXqnj+9hDwtYj4YVuyeoQZvefeluavGpo41Wdcdb/66N33JONLF6XvKR9W+tnuv3jTRxvGXvmr05LzTv/BVcl46pn0ALFnT+NYi0NRDy5cmIyPbt7cUvudMOlij4jbgD9uYy5m1kHuejPLhIvdLBMudrNMuNjNMuFiN8uEb3Hthmjtdsp44IF0PDWkc4tDEx8+d0Myvn4k/ajqP/nROxu3/atbk/OOVuVecZtqqnut6vHcVV1z/di1VsVbdrNMuNjNMuFiN8uEi90sEy52s0y42M0y4WI3y4T72bsg2Q8ODC6Yn4yP3rNl0ssenD+vpbZ/95YnJ+MnfnBpMn7EP97VMDb2QHpY5KprAKqGVU423eItrlORt+xmmXCxm2XCxW6WCRe7WSZc7GaZcLGbZcLFbpYJ97P3gaq+7sE5c9INPGpG47Y3bkrOWnVf98BtjfvJARa+Lf2Y69ideJzz6GhyXmsvb9nNMuFiN8uEi90sEy52s0y42M0y4WI3y4SL3SwT7mfvgoFZs5Lx2LM3GR/dti29gKp4giqezT56b8Vw0xXxgRmNrwFIDancFgOJ4aTH8uvjr9yySzpX0iZJN9RMmy/pEkm/L7+nn5BgZj3XzG78auD4cdPeC1waEYcBl5Y/m1kfqyz2iLgMGH8954nAeeXr84CT2pyXmbXZZI/ZF0XE+vL1BmBRozdKWgmsBJjBzEkuzsxa1fLZ+IgIoOGTASNiVUQsi4hlw6RvujCzzplssW+UtBig/J6+tcrMem6yxX4RcGr5+lTgO+1Jx8w6pfKYXdL5wArgQElrgQ8AZwEXSHojcAdwcieTnOqq+tE1bTg9/97J90cPzqt4bnxFP/ng4Ycl49q6PRkfWZ8e370lqX50e5jKYo+IUxqEjmtzLmbWQb5c1iwTLnazTLjYzTLhYjfLhIvdLBO+xbULqrrOquJVj5Ie27mzYayqay11CyrA5o+mb4G9/4onJONLPtTBrjebEG/ZzTLhYjfLhIvdLBMudrNMuNjNMuFiN8uEi90sE+5n7wcVt2qO7rg/PX8Lj0Ue27UrGZ8zIx3f8pQHkvFUP/7Y7t3JeYmGD0AqG6j4vX0L7H68ZTfLhIvdLBMudrNMuNjNMuFiN8uEi90sEy52s0y4n70PaDj9Z4iq/uiEwYULk/HRzZuT8b2fOSgZP/S+kWRcs2c3jlX0o7fyewNZDsuc4i27WSZc7GaZcLGbZcLFbpYJF7tZJlzsZplwsZtlwv3sXTAwc2YynnruO4CGKvrhRxr3dVf1o+85/lnJ+P2L0veEz77ijmSc6dMahjSYbrvibnaboMotu6RzJW2SdEPNtDMlrZN0bfl1QmfTNLNWNbMbvxo4vs70T0TE0eXXxe1Ny8zarbLYI+IyYEsXcjGzDmrlBN3bJV1X7ubPa/QmSSslrZG0Zi8tXutsZpM22WL/HPBE4GhgPXB2ozdGxKqIWBYRy4aZPsnFmVmrJlXsEbExIkYjYgz4ArC8vWmZWbtNqtglLa758RXADY3ea2b9obKfXdL5wArgQElrgQ8AKyQdTdEVejvwlg7mOOVV9aNXjZFe9Wx3DTfuy64a+31kZvrzftf89PjsIxs2JuPWPyqLPSJOqTP5nA7kYmYd5MtlzTLhYjfLhIvdLBMudrNMuNjNMuFbXLtA09NXDrbStQYQo40fmVx1e+26FyTDzP9N+kbTgVmzkvGx+yuGm7au8ZbdLBMudrNMuNjNMuFiN8uEi90sEy52s0y42M0y4X72bkj0gwMwUPFI5YrbVFN93fGUpel556cfFTZrQ7qP3/3oU4e37GaZcLGbZcLFbpYJF7tZJlzsZplwsZtlwsVulgn3s3eDKj5Tx9L96INz5iTjo9u2NYzd9YK5yXmvfl7DwXwAeMn335mMpx+Cbf3EW3azTLjYzTLhYjfLhIvdLBMudrNMuNjNMuFiN8tEM0M2LwG+BCyiGKJ5VUR8StJ84BvAUophm0+OiHs7l+rUVXU/+tCSxyXjI3eunfSyZ9ydfu7707+Z7kefNys9ZLNNHc1s2UeA0yPiCOAY4G2SjgDeC1waEYcBl5Y/m1mfqiz2iFgfEdeUr7cDNwMHAycC55VvOw84qVNJmlnrJnTMLmkp8HTgSmBRRKwvQxsodvPNrE81XeySDgC+CZwWEftdjB0RQXE8X2++lZLWSFqzl/Tzzsysc5oqdknDFIX+1Yj4Vjl5o6TFZXwxsKnevBGxKiKWRcSyYdIDHJpZ51QWuyQB5wA3R8THa0IXAaeWr08FvtP+9MysXZq5xfW5wOuA6yVdW047AzgLuEDSG4E7gJM7k+LUVzVkc1XX2tDig5Lxbccc0jA2NpyclT864/pkvOpR0YMHLkjGR+++J52AdU1lsUfE5UCjztbj2puOmXWKr6Azy4SL3SwTLnazTLjYzTLhYjfLhIvdLBN+lHQXDMycmYyP7k5fRjyyfkMyPvTWxv34h85I95Pv/E7j4Z4BeGBXMux+9KnDW3azTLjYzTLhYjfLhIvdLBMudrNMuNjNMuFiN8uE+9m7YPTeiidsK/245qHHpx81/btbGz/+b8st6T/xQRt/mYxrKD1/jCXD1ke8ZTfLhIvdLBMudrNMuNjNMuFiN8uEi90sEy52s0y4n70bKvrRifSwyrEzfU/5QYc0vqf80CO3JOfdeP0zk/Hp19yWjFdeQ2B9w1t2s0y42M0y4WI3y4SL3SwTLnazTLjYzTLhYjfLRGU/u6QlwJeARUAAqyLiU5LOBN4MbC7fekZEXNypRKe0in70gdmzk/HRzZuT8bknbWsY2zr7gOS8Q3dfnYyzcGE6blNGMxfVjACnR8Q1kmYDV0u6pIx9IiI+1rn0zKxdKos9ItYD68vX2yXdDBzc6cTMrL0mdMwuaSnwdODKctLbJV0n6VxJ8xrMs1LSGklr9pIe5sjMOqfpYpd0APBN4LSI2AZ8DngicDTFlv/sevNFxKqIWBYRy4ZpPCaZmXVWU8UuaZii0L8aEd8CiIiNETEaEWPAF4DlnUvTzFpVWeySBJwD3BwRH6+Zvrjmba8Abmh/embWLs2cjX8u8DrgeknXltPOAE6RdDRFd9ztwFs6kqFVisSQz1XDQVe2vX17S/Nb/2jmbPzlQL0bst2nbjaF+Ao6s0y42M0y4WI3y4SL3SwTLnazTLjYzTLhR0l3gYanJeNjFX3ZVcMmD8ydk2g8fXvt6H2Nb48FGNuVfoz1wMyZ6fl37kzGrXu8ZTfLhIvdLBMudrNMuNjNMuFiN8uEi90sEy52s0woKh5z3NaFSZuBO2omHQjc3bUEJqZfc+vXvMC5TVY7czskIuo+/7urxf6whUtrImJZzxJI6Nfc+jUvcG6T1a3cvBtvlgkXu1kmel3sq3q8/JR+za1f8wLnNlldya2nx+xm1j293rKbWZe42M0y0ZNil3S8pN9K+oOk9/Yih0Yk3S7peknXSlrT41zOlbRJ0g010+ZLukTS78vvdcfY61FuZ0paV667ayWd0KPclkj6maSbJN0o6R3l9J6uu0ReXVlvXT9mlzQI/A54EbAWuAo4JSJu6moiDUi6HVgWET2/AEPS84AdwJci4mnltI8AWyLirPKDcl5E/H2f5HYmsKPXw3iXoxUtrh1mHDgJeD09XHeJvE6mC+utF1v25cAfIuK2iNgDfB04sQd59L2IuAzYMm7yicB55evzKP5Zuq5Bbn0hItZHxDXl6+3AvmHGe7ruEnl1RS+K/WDgzpqf19Jf470H8GNJV0ta2etk6lgUEevL1xuARb1Mpo7KYby7adww432z7iYz/HmrfILu4Y6NiGcALwXeVu6u9qUojsH6qe+0qWG8u6XOMOMP6uW6m+zw563qRbGvA5bU/Py4clpfiIh15fdNwIX031DUG/eNoFt+39TjfB7UT8N41xtmnD5Yd70c/rwXxX4VcJikQyVNA14FXNSDPB5G0qzyxAmSZgEvpv+Gor4IOLV8fSrwnR7msp9+Gca70TDj9Hjd9Xz484jo+hdwAsUZ+VuB9/UihwZ5PQH4Tfl1Y69zA86n2K3bS3Fu443AAuBS4PfAT4D5fZTbl4HrgesoCmtxj3I7lmIX/Trg2vLrhF6vu0ReXVlvvlzWLBM+QWeWCRe7WSZc7GaZcLGbZcLFbpYJF7tZJlzsZpn4f79K2hZMVI75AAAAAElFTkSuQmCC\n", | |
| "text/plain": [ | |
| "<Figure size 432x288 with 1 Axes>" | |
| ] | |
| }, | |
| "metadata": { | |
| "needs_background": "light" | |
| } | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "C0LAibq1OtPN" | |
| }, | |
| "source": [ | |
| "## From a dataframe" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/" | |
| }, | |
| "id": "FMNAfrLlHlFs", | |
| "outputId": "1626917f-b9da-4867-8900-a7a0a3996454" | |
| }, | |
| "source": [ | |
| "DataGen = ImageDataGenerator(rescale=1./255)\n", | |
| "gen = DataGen.flow_from_dataframe(pd.DataFrame({'filename': fpaths}), \n", | |
| " batch_size=batch_size, shuffle=True, seed=10, \n", | |
| " class_mode=None, target_size=(28, 28), \n", | |
| " color_mode='grayscale')" | |
| ], | |
| "execution_count": 65, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "name": "stdout", | |
| "text": [ | |
| "Found 10000 validated image filenames.\n" | |
| ] | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/" | |
| }, | |
| "id": "ZlnOCIfgHsnJ", | |
| "outputId": "c8311992-f44d-4361-ebad-34e2348299d9" | |
| }, | |
| "source": [ | |
| "%%time\n", | |
| "for i in range(n_batches * epochs):\n", | |
| " n = next(gen)" | |
| ], | |
| "execution_count": 66, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "name": "stdout", | |
| "text": [ | |
| "CPU times: user 3.22 s, sys: 264 ms, total: 3.48 s\n", | |
| "Wall time: 3.5 s\n" | |
| ] | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 298 | |
| }, | |
| "id": "pz13gmg_PUI0", | |
| "outputId": "c947f08a-b80b-4347-ab64-2a1b3ac47c18" | |
| }, | |
| "source": [ | |
| "plt.title(f'last batch out shape: {n.shape}')\n", | |
| "plt.imshow(n[0].squeeze())" | |
| ], | |
| "execution_count": 67, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "data": { | |
| "text/plain": [ | |
| "<matplotlib.image.AxesImage at 0x7f6f842259d0>" | |
| ] | |
| }, | |
| "metadata": {}, | |
| "execution_count": 67 | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAEICAYAAACZA4KlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAXcklEQVR4nO3df7RdZX3n8ffn3tzkkhBCAhIiIvEHXf4oTrRpiqvUpoM6iGsVbGdR0bbYcRpnqjO4SnUcujqypj+WtaXWmVptWmmQIpRORam1tRSZIu2URUTkh0hBDCUxJECA/CK/7vnOH3sHDpd7nufes8+v5Pm81jrrnrOfs/f+nn3P9+wfz36eRxGBmR39xoYdgJkNhpPdrBBOdrNCONnNCuFkNyuEk92sEENPdkmbJL15AOsJSa8cwHo2SPqNfq+nCUlrJW0edhwzkbRA0rclrRh2LKNK0uWS/vNc5xt6svdCvxNsUD9IczHKCdvQOuCWiNgKzyb/ZyRtk7RD0l9JOmU2C5J0pqQb6/kek/QX7T8iI7zs+ZL+T/29C0lrp73ld4FLJc2fzfIOOyqS3Y4q/wm4qu31xcAbgdcBLwaeBP73LJe1FFgPrAROA3YBf3oELBvgVuBngUenF9Q/hN8BfnIOy4OIGOoD2AS8uX6+Bvh/wFPAVuAPgPl1mYBPANuBncDdwA9S7QkOAgeA3cBfdVhPAP8VeAh4HPgdYKwuewXwNeCJuuxq4Pi67CqgBTxTL//D9fSzgH+qY30EeE89fQPwKeCvqb4AtwGvSHz+nwTurZfzf4FXT4v5lW2vNwC/ASyq42nVMe0GXjzDss8Fvl3HsQX4lXr6WmAzcEm9PbcCv9A239uBb9bb+RHgsraylXVc64Dv1/P+Slv5GPAR4Lv19rwOWDbL78JL6881r23ap4GPT4vt/i6/a28Adh1Jy67/T2tnmP6rwJ/OaVn9SOA5fphNPJfsPwScCcyrv1T3AR+sy/4d8A3geKrEfzWwoj0JMusJ4GZgWf2l+hfgP9ZlrwTeAiwAXgTcAvz+TDHWrw//kl8ITAAnAKvaYnmC6odrHtUPx7UdYvoBYE+97gngw8CDPPcDN2Oy18/XApszn3kr8GP186XAG9rmPQT8z3q95wJ7gaVt5WdQJe7rgG3A+XXZyjqua6h+dM4AHmv7H14M/DPwknp7/hFwTVtMdwHv6hDv24F7p01bDfwj1d5xIfD59v/NHL9rHwT++UhaNp2T/aeAO+a0rH4n8yw+zPMSaYYNeH39/N9SJeiZ1HvkmZIgsZ4Azml7/UvATR3eez7wzU4xAv/9cFwzzLsB+JO21+cC3+nw3l8Drmt7PUa1B17bFnOTZP9X4H3AcdOmr+WFe9DtwJkdlvP7wCfq54eT/VVt5R8HPls/vw84u61sBdWR17xUrPV7392eMPW0JcC19ToPUR1xzOpIYdpyXgfsoP7xO4KW3SnZ3wI8NJdljdQ5u6QfkPRlSY9K2gn8FnAiQER8jeqw/lPAdknrJR03x1U80vb8YapfXSQtl3StpC31ev/s8Ho7OJXqMLWT9vOsvcCxHd734joOACKiVcc4qws5s/DTVD82D0v6B0lvbCt7IiIOzRSnpB+RdHN94elpqvPo6dtjxm1JddRzvaSnJD1FlfxTwPJZxPsksHjatE9RHSGcQHUk8QXgb2axrGfVtTB/A1wcEV8/UpadsZjq1G/WRirZqc5zvgOcHhHHAZdSHbIDEBH/KyJ+CHgN1SHwhw4XzXL5p7Y9fynVOSdUPyoBnFGv92fb1zvD8h+hOs9v6vtUyQGAJNUxbqkn7aU6BDzs5ERMLxARt0fEecBJwBepzp9n4/PADcCpEbEE+AzP3x7QeVs+ArwtIo5ve0xGxBby7gJeJmle27RVwIaI2BER+6kucq2RlPoxfpak04C/B349Iq6aVjyyy56FVwPfmssMo5bsi6kuCu2W9Crg2bpEST9c73EmqM5z91FdoILqnPLls1j+hyQtlXQq1bnln7etdzfwdF098qFp801f/tXAmyVdIGmepBMkrZrTJ61cB7xd0tn157oE2E914Q/gTuBdksYlnQP8+LSYTpC0ZKYF19U375a0JCIOUm3X1kzvncFiYEdE7JO0BnjXDO/5NUkLJb0W+AWe25afAX6zTgQkvUjSebNZaURsprpmsaZt8u3Az0taUm+jXwK+HxGP18vfIGnDTMur/5dfA/4gIj4zw1tGctl1+QJJk/XL+ZIm653BYT/OXI8U5noO0esHz79A9yaqPftu4OtUF5BurcvOpvrl381zV8yPrctOp0qMp4AvdlhP+9X4J4DLgfG67LVUF/9218u5hLbzYeA8qvPfp3juivaPUV1pP3zF+qJ6+gbarh+QObcG3kF1xfxp4B+A17aVraa6Ur+LqlbgmmnLvqL+LE8x7Wo8MB/4W6pD451UX76zOsU07f/w76kOzXcBX6Y6ffqzumwlz78a/yh1DUVdPgb8MnB/Pf93gd9qK78XeHdie7wf+HTb6xPq//X2+nPeCqxpK78J+MUOy/poHevu9seoL7vt/xHTHivrshVU5/Lz55Jrqmc2mxVJK4HvARPx/HP+Xi1/AdXFrLOjvrEm8d75VIeyr6uPXnoZx8guW9LlwHcj4g/nNJ+T3eai38lu/TNq5+xm1ifes5sVwnt2s0LMy7+ld+ZrQUyyaJCrLILGuv/NjtZsa+O68/zaomnrPpKPKjt/rMqQPto+9nAg9s8YXaNkr+t+PwmMU90i+rHU+ydZxI/o7CarPDIlvvAA5L70mfnHjjmm63lbe/ak193Q2ORkx7LWvn19XXc/aV46deLQcK5d3hY3dSzrepcgaZzqlsC3Ud3RdqGk13S7PDPrrybn7GuAByPioYg4QHXT/6zulDKzwWuS7Kfw/MYQm5mhAYekdZI2Stp4kP0NVmdmTfT9anxErI+I1RGxeoIF/V6dmXXQJNm38PyWTy/hudZaZjZimiT77cDpkl5W3+v7TqpmkWY2grqueouIQ5I+AHyVqurtioi4t2eRHU2a1idn5m/t3dv1ojWR7qBUE5kqpoPpKqZ+Vq/lqr8YH+9YlIub1lSyeFhVa000qmePiK8AX+lRLGbWR75d1qwQTnazQjjZzQrhZDcrhJPdrBBOdrNCDLQ9e6m0IHOb8FS6Tjc3vyY7l0/teDI5bxw80Kh8bFG6f4Lc/E1k67pbne9PUKIOHiAy9exHIu/ZzQrhZDcrhJPdrBBOdrNCONnNCuFkNyuEq94GIPY3644rW8XUoIfYeSenh02Pvc8ky6d27kyvING7reZNpNd9KDMMWq7pcKL6LFu11rRH4BHkPbtZIZzsZoVwspsVwsluVggnu1khnOxmhXCymxXC9ewDMH7CsmR57El3BZ2tZ091mZyp4z/06Lb0sjPGFi9Olrd27UrM3GB0WqC1L3P/QqouPVOPnmsCi9L7yX427e2W9+xmhXCymxXCyW5WCCe7WSGc7GaFcLKbFcLJblYI17MPwNQTO/q6fCXqfHPdUDdua/9Mur17k3VHpi58LPfZplJDNme60M7d25Br7z6CGiW7pE3ALmAKOBQRq3sRlJn1Xi/27D8REY/3YDlm1kc+ZzcrRNNkD+DvJH1D0rqZ3iBpnaSNkjYepNn5oZl1r+lh/FkRsUXSScCNkr4TEbe0vyEi1gPrAY7TsiOvlz6zo0SjPXtEbKn/bgeuB9b0Iigz672uk13SIkmLDz8H3grc06vAzKy3mhzGLweuV1XfOA/4fET8bU+iKsy8FScny3NtzlN1xrsvODM572kX358s37F/YbL88atfmiw/6Yudl5+9/yDTN3srU0+f65e+kSOw3/iukz0iHgL+TQ9jMbM+ctWbWSGc7GaFcLKbFcLJblYIJ7tZIdzEdQS0dia6W4ZsNY8m5ncs23la+vf8PctvTZa/dWF62OTv/Y/dyfJ1D/6XjmUT96abiU49/kSyfJjVX2OLFiXLWw2G0e4X79nNCuFkNyuEk92sEE52s0I42c0K4WQ3K4ST3awQrmcfgPHjlyTLp556Olk+NjmZLG/t29ex7JSbdybn/fLPrEqWv3Xh7cnyyUyPyot/fXPHsj1vytSjNxSHOt8joHmZr35myOZRrEfP8Z7drBBOdrNCONnNCuFkNyuEk92sEE52s0I42c0K4Xr2AcjVo+eMnXxSsry16V87lsXGdFf+f/31dFfTH/7pm5Ply8ePSZa/++TbOpb9yaIzkvMylt4XtXZ13w9AdkjmVqatfG7I5hHsatp7drNCONnNCuFkNyuEk92sEE52s0I42c0K4WQ3K4Tr2Y8ArSee7NuyJx9r9ns/oXS778Vjz3Qsy7UJH1u8OF2+MD2cdGvv3mR5euapdHmunn0EZf/Tkq6QtF3SPW3Tlkm6UdID9d+l/Q3TzJqazc/6BuCcadM+AtwUEacDN9WvzWyEZZM9Im4BdkybfB5wZf38SuD8HsdlZj3W7Tn78ojYWj9/FFje6Y2S1gHrACZJn2OZWf80vhofEQF0vOs/ItZHxOqIWD3BgqarM7MudZvs2yStAKj/bu9dSGbWD90m+w3ARfXzi4Av9SYcM+uX7Dm7pGuAtcCJkjYDHwU+Blwn6b3Aw8AF/QzySJcaPx0gDh5IlmfbbSfqfMczddWHFqbbXb9k3rHJ8r2tdOyTDU4UW3vS9eSaaHCbyFj6/oBsPfsRKLu1IuLCDkVn9zgWM+sj3y5rVggnu1khnOxmhXCymxXCyW5WCDdxHYBc1VrTqrlUU884kJ53Yle6qebBaFYF9aqJRDPWXPVXtNLF+/d3EVGtYdXa2IL03aCpYbSHxXt2s0I42c0K4WQ3K4ST3awQTnazQjjZzQrhZDcrhOvZByE7vG+6Pjkn1yVzijIjC4+Rjn3hWPoegd9+7PWdlz2Zqat+pnM31P2meenUGMV69Bzv2c0K4WQ3K4ST3awQTnazQjjZzQrhZDcrhJPdrBCuZx8AzZtIlufaq/fTgePSFe3jarY/uOqWszqWnb73tkbLzhk/fknHsqmnnk7Om6tnj0OHuoppmLxnNyuEk92sEE52s0I42c0K4WQ3K4ST3awQTnazQriefQCy9eiZ9u7jy5Ymy6ee2DHXkJ41djC97q2HdifLF4+lv0ILt3TuG77fddmtPd23h9f8dDt9jsb27JKukLRd0j1t0y6TtEXSnfXj3P6GaWZNzeYwfgNwzgzTPxERq+rHV3oblpn1WjbZI+IWoPvjRDMbCU0u0H1A0l31YX7Hk0pJ6yRtlLTxIA3G5jKzRrpN9k8DrwBWAVuByzu9MSLWR8TqiFg9QbqDQTPrn66SPSK2RcRURLSAPwbW9DYsM+u1rpJd0oq2l+8A7un0XjMbDdl6dknXAGuBEyVtBj4KrJW0CghgE/C+PsZ49ItM5+1T3Y8lrtw44hPpdS/J9AvfIt3n/cHFieU3bCufradP3N+gifTnmtq5s6uYRlk22SPiwhkmf7YPsZhZH/l2WbNCONnNCuFkNyuEk92sEE52s0K4iesgjHVu5glAK121FgcOdr3q2J++RXn8mXQT11zV2gKlu8le8kDnsmzT38x2yzVDTTWR1UT31XZwZHY17T27WSGc7GaFcLKbFcLJblYIJ7tZIZzsZoVwspsVwvXsgxDpuuqc1jOZLpFT9dGZOvycY8cmk+W7W+kulU/8x20dy5pFBnGg+6Guo0GzYYBoZZoljyDv2c0K4WQ3K4ST3awQTnazQjjZzQrhZDcrhJPdrBCuZx+EXFfRmSGbc/NrPDF/ZkjlyDS1z/nq3pOS5VMPPNRsBQmN2owfgfXkTXnPblYIJ7tZIZzsZoVwspsVwsluVggnu1khnOxmhZjNkM2nAp8DllMN0bw+Ij4paRnw58BKqmGbL4iIJ/sX6tFL4+nK7lx9crJ/9Ez/5gsy/7GnW+m29J/83juS5cfwvY5ludhyQzpHpq1+avman+7vPttvfK7f+f1NW+v33mz27IeASyLiNcCZwPslvQb4CHBTRJwO3FS/NrMRlU32iNgaEXfUz3cB9wGnAOcBV9ZvuxI4v19Bmllzczpnl7QSeD1wG7A8IrbWRY9SHeab2YiadbJLOhb4S+CDEbGzvSwigup8fqb51knaKGnjQdLjjplZ/8wq2SVNUCX61RHxhXryNkkr6vIVwPaZ5o2I9RGxOiJWT7CgFzGbWReyyS5JwGeB+yLi99qKbgAuqp9fBHyp9+GZWa/MponrjwI/B9wt6c562qXAx4DrJL0XeBi4oD8hHv2y3RI3GPI5V2134t3prqC/deCYZPmW7ccny1+ZiD035HK2C+2M5GfPVOtll92gG+thySZ7RNwKdGowfXZvwzGzfvEddGaFcLKbFcLJblYIJ7tZIZzsZoVwspsVwl1JD4AWpO8cjP3p24jHFi5Mlrf27p1zTIdN3H5/svw4pWM746XfT5Y/k7gHIFuPnuuCu4FcE9am3XuPIu/ZzQrhZDcrhJPdrBBOdrNCONnNCuFkNyuEk92sEK5nH4BsV9G5+XNdLufauye09uxJlq/K3CPwm6d9MVn+y7yxc2HDumpNpNvDZ+vSU8vOtLXP3RsxirxnNyuEk92sEE52s0I42c0K4WQ3K4ST3awQTnazQriefQBy7c3Hly5Nlk892WAk7Ey77LHJyWT5629/Z7L8mz987ZxD6pUm9ejZZR+B9eg53rObFcLJblYIJ7tZIZzsZoVwspsVwsluVggnu1khsvXskk4FPgcsp2p6vT4iPinpMuAXgcfqt14aEV/pV6BHs0b16Bm5tvStfenx2fnqsmTxy5/+D8ny07kjvXwbmNncVHMIuCQi7pC0GPiGpBvrsk9ExO/2Lzwz65VsskfEVmBr/XyXpPuAU/odmJn11pzO2SWtBF4P3FZP+oCkuyRdIWnGez4lrZO0UdLGgxx9tyCaHSlmneySjgX+EvhgROwEPg28AlhFtee/fKb5ImJ9RKyOiNUTpPszM7P+mVWyS5qgSvSrI+ILABGxLSKmIqIF/DGwpn9hmllT2WSXJOCzwH0R8Xtt01e0ve0dwD29D8/MemU2V+N/FPg54G5Jd9bTLgUulLSKqjpuE/C+vkR4FBhbtChZnmtO2WTI52g16675pD/8p2T5ir94UbK884DNNmizuRp/KzBTo2jXqZsdQXwHnVkhnOxmhXCymxXCyW5WCCe7WSGc7GaFcFfSA5DrSjo3dHFuyOY4dKjreTU/3ZV0rgns1GOPJcttdHjPblYIJ7tZIZzsZoVwspsVwsluVggnu1khnOxmhVBk6nh7ujLpMeDhtkknAo8PLIC5GdXYRjUucGzd6mVsp0XEjJ0MDDTZX7ByaWNErB5aAAmjGtuoxgWOrVuDis2H8WaFcLKbFWLYyb5+yOtPGdXYRjUucGzdGkhsQz1nN7PBGfae3cwGxMluVoihJLukcyTdL+lBSR8ZRgydSNok6W5Jd0raOORYrpC0XdI9bdOWSbpR0gP13xnH2BtSbJdJ2lJvuzslnTuk2E6VdLOkb0u6V9LF9fShbrtEXAPZbgM/Z5c0DvwL8BZgM3A7cGFEfHuggXQgaROwOiKGfgOGpDcBu4HPRcQP1tM+DuyIiI/VP5RLI+K/jUhslwG7hz2Mdz1a0Yr2YcaB84H3MMRtl4jrAgaw3YaxZ18DPBgRD0XEAeBa4LwhxDHyIuIWYMe0yecBV9bPr6T6sgxch9hGQkRsjYg76ue7gMPDjA912yXiGohhJPspwCNtrzczWuO9B/B3kr4had2wg5nB8ojYWj9/FFg+zGBmkB3Ge5CmDTM+Mtuum+HPm/IFuhc6KyLeALwNeH99uDqSojoHG6W601kN4z0oMwwz/qxhbrtuhz9vahjJvgU4te31S+ppIyEittR/twPXM3pDUW87PIJu/Xf7kON51igN4z3TMOOMwLYb5vDnw0j224HTJb1M0nzgncANQ4jjBSQtqi+cIGkR8FZGbyjqG4CL6ucXAV8aYizPMyrDeHcaZpwhb7uhD38eEQN/AOdSXZH/LvCrw4ihQ1wvB75VP+4ddmzANVSHdQeprm28FzgBuAl4APh7YNkIxXYVcDdwF1VirRhSbGdRHaLfBdxZP84d9rZLxDWQ7ebbZc0K4Qt0ZoVwspsVwsluVggnu1khnOxmhXCymxXCyW5WiP8P9Uwhmc6b90UAAAAASUVORK5CYII=\n", | |
| "text/plain": [ | |
| "<Figure size 432x288 with 1 Axes>" | |
| ] | |
| }, | |
| "metadata": { | |
| "needs_background": "light" | |
| } | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "gnZqc5oqOxah" | |
| }, | |
| "source": [ | |
| "## From a memmap" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "-Ld6aurlLO9c" | |
| }, | |
| "source": [ | |
| "# Loading the memmap from disk (does not load all the data to RAM), shape must be specified\n", | |
| "mmap = np.memmap('mnist.mmap', dtype='uint8', mode='r', shape=(x_test.shape))\n", | |
| "\n", | |
| "DataGen = ImageDataGenerator(rescale=1./255)\n", | |
| "gen = DataGen.flow(mmap, y=None, batch_size=batch_size, shuffle=True, seed=10)" | |
| ], | |
| "execution_count": 68, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/" | |
| }, | |
| "id": "M_kloBV0MU2q", | |
| "outputId": "b1a49025-2b20-4101-c03c-e4a7b4c0c653" | |
| }, | |
| "source": [ | |
| "%%time\n", | |
| "for i in range(n_batches * epochs):\n", | |
| " n = next(gen)" | |
| ], | |
| "execution_count": 69, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "name": "stdout", | |
| "text": [ | |
| "CPU times: user 294 ms, sys: 0 ns, total: 294 ms\n", | |
| "Wall time: 296 ms\n" | |
| ] | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 298 | |
| }, | |
| "id": "mqP74kDbOWvz", | |
| "outputId": "2976d4a4-8fed-454a-cfc9-bf356151a6ae" | |
| }, | |
| "source": [ | |
| "plt.title(f'last batch out shape: {n.shape}')\n", | |
| "plt.imshow(n[0].squeeze())" | |
| ], | |
| "execution_count": 70, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "data": { | |
| "text/plain": [ | |
| "<matplotlib.image.AxesImage at 0x7f6f83ed0710>" | |
| ] | |
| }, | |
| "metadata": {}, | |
| "execution_count": 70 | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAEICAYAAACZA4KlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAXcklEQVR4nO3df7RdZX3n8ffn3tzkkhBCAhIiIvEHXf4oTrRpiqvUpoM6iGsVbGdR0bbYcRpnqjO4SnUcujqypj+WtaXWmVptWmmQIpRORam1tRSZIu2URUTkh0hBDCUxJECA/CK/7vnOH3sHDpd7nufes8+v5Pm81jrrnrOfs/f+nn3P9+wfz36eRxGBmR39xoYdgJkNhpPdrBBOdrNCONnNCuFkNyuEk92sEENPdkmbJL15AOsJSa8cwHo2SPqNfq+nCUlrJW0edhwzkbRA0rclrRh2LKNK0uWS/vNc5xt6svdCvxNsUD9IczHKCdvQOuCWiNgKzyb/ZyRtk7RD0l9JOmU2C5J0pqQb6/kek/QX7T8iI7zs+ZL+T/29C0lrp73ld4FLJc2fzfIOOyqS3Y4q/wm4qu31xcAbgdcBLwaeBP73LJe1FFgPrAROA3YBf3oELBvgVuBngUenF9Q/hN8BfnIOy4OIGOoD2AS8uX6+Bvh/wFPAVuAPgPl1mYBPANuBncDdwA9S7QkOAgeA3cBfdVhPAP8VeAh4HPgdYKwuewXwNeCJuuxq4Pi67CqgBTxTL//D9fSzgH+qY30EeE89fQPwKeCvqb4AtwGvSHz+nwTurZfzf4FXT4v5lW2vNwC/ASyq42nVMe0GXjzDss8Fvl3HsQX4lXr6WmAzcEm9PbcCv9A239uBb9bb+RHgsraylXVc64Dv1/P+Slv5GPAR4Lv19rwOWDbL78JL6881r23ap4GPT4vt/i6/a28Adh1Jy67/T2tnmP6rwJ/OaVn9SOA5fphNPJfsPwScCcyrv1T3AR+sy/4d8A3geKrEfzWwoj0JMusJ4GZgWf2l+hfgP9ZlrwTeAiwAXgTcAvz+TDHWrw//kl8ITAAnAKvaYnmC6odrHtUPx7UdYvoBYE+97gngw8CDPPcDN2Oy18/XApszn3kr8GP186XAG9rmPQT8z3q95wJ7gaVt5WdQJe7rgG3A+XXZyjqua6h+dM4AHmv7H14M/DPwknp7/hFwTVtMdwHv6hDv24F7p01bDfwj1d5xIfD59v/NHL9rHwT++UhaNp2T/aeAO+a0rH4n8yw+zPMSaYYNeH39/N9SJeiZ1HvkmZIgsZ4Azml7/UvATR3eez7wzU4xAv/9cFwzzLsB+JO21+cC3+nw3l8Drmt7PUa1B17bFnOTZP9X4H3AcdOmr+WFe9DtwJkdlvP7wCfq54eT/VVt5R8HPls/vw84u61sBdWR17xUrPV7392eMPW0JcC19ToPUR1xzOpIYdpyXgfsoP7xO4KW3SnZ3wI8NJdljdQ5u6QfkPRlSY9K2gn8FnAiQER8jeqw/lPAdknrJR03x1U80vb8YapfXSQtl3StpC31ev/s8Ho7OJXqMLWT9vOsvcCxHd734joOACKiVcc4qws5s/DTVD82D0v6B0lvbCt7IiIOzRSnpB+RdHN94elpqvPo6dtjxm1JddRzvaSnJD1FlfxTwPJZxPsksHjatE9RHSGcQHUk8QXgb2axrGfVtTB/A1wcEV8/UpadsZjq1G/WRirZqc5zvgOcHhHHAZdSHbIDEBH/KyJ+CHgN1SHwhw4XzXL5p7Y9fynVOSdUPyoBnFGv92fb1zvD8h+hOs9v6vtUyQGAJNUxbqkn7aU6BDzs5ERMLxARt0fEecBJwBepzp9n4/PADcCpEbEE+AzP3x7QeVs+ArwtIo5ve0xGxBby7gJeJmle27RVwIaI2BER+6kucq2RlPoxfpak04C/B349Iq6aVjyyy56FVwPfmssMo5bsi6kuCu2W9Crg2bpEST9c73EmqM5z91FdoILqnPLls1j+hyQtlXQq1bnln7etdzfwdF098qFp801f/tXAmyVdIGmepBMkrZrTJ61cB7xd0tn157oE2E914Q/gTuBdksYlnQP8+LSYTpC0ZKYF19U375a0JCIOUm3X1kzvncFiYEdE7JO0BnjXDO/5NUkLJb0W+AWe25afAX6zTgQkvUjSebNZaURsprpmsaZt8u3Az0taUm+jXwK+HxGP18vfIGnDTMur/5dfA/4gIj4zw1tGctl1+QJJk/XL+ZIm653BYT/OXI8U5noO0esHz79A9yaqPftu4OtUF5BurcvOpvrl381zV8yPrctOp0qMp4AvdlhP+9X4J4DLgfG67LVUF/9218u5hLbzYeA8qvPfp3juivaPUV1pP3zF+qJ6+gbarh+QObcG3kF1xfxp4B+A17aVraa6Ur+LqlbgmmnLvqL+LE8x7Wo8MB/4W6pD451UX76zOsU07f/w76kOzXcBX6Y6ffqzumwlz78a/yh1DUVdPgb8MnB/Pf93gd9qK78XeHdie7wf+HTb6xPq//X2+nPeCqxpK78J+MUOy/poHevu9seoL7vt/xHTHivrshVU5/Lz55Jrqmc2mxVJK4HvARPx/HP+Xi1/AdXFrLOjvrEm8d75VIeyr6uPXnoZx8guW9LlwHcj4g/nNJ+T3eai38lu/TNq5+xm1ifes5sVwnt2s0LMy7+ld+ZrQUyyaJCrLILGuv/NjtZsa+O68/zaomnrPpKPKjt/rMqQPto+9nAg9s8YXaNkr+t+PwmMU90i+rHU+ydZxI/o7CarPDIlvvAA5L70mfnHjjmm63lbe/ak193Q2ORkx7LWvn19XXc/aV46deLQcK5d3hY3dSzrepcgaZzqlsC3Ud3RdqGk13S7PDPrrybn7GuAByPioYg4QHXT/6zulDKzwWuS7Kfw/MYQm5mhAYekdZI2Stp4kP0NVmdmTfT9anxErI+I1RGxeoIF/V6dmXXQJNm38PyWTy/hudZaZjZimiT77cDpkl5W3+v7TqpmkWY2grqueouIQ5I+AHyVqurtioi4t2eRHU2a1idn5m/t3dv1ojWR7qBUE5kqpoPpKqZ+Vq/lqr8YH+9YlIub1lSyeFhVa000qmePiK8AX+lRLGbWR75d1qwQTnazQjjZzQrhZDcrhJPdrBBOdrNCDLQ9e6m0IHOb8FS6Tjc3vyY7l0/teDI5bxw80Kh8bFG6f4Lc/E1k67pbne9PUKIOHiAy9exHIu/ZzQrhZDcrhJPdrBBOdrNCONnNCuFkNyuEq94GIPY3644rW8XUoIfYeSenh02Pvc8ky6d27kyvING7reZNpNd9KDMMWq7pcKL6LFu11rRH4BHkPbtZIZzsZoVwspsVwsluVggnu1khnOxmhXCymxXC9ewDMH7CsmR57El3BZ2tZ091mZyp4z/06Lb0sjPGFi9Olrd27UrM3GB0WqC1L3P/QqouPVOPnmsCi9L7yX427e2W9+xmhXCymxXCyW5WCCe7WSGc7GaFcLKbFcLJblYI17MPwNQTO/q6fCXqfHPdUDdua/9Mur17k3VHpi58LPfZplJDNme60M7d25Br7z6CGiW7pE3ALmAKOBQRq3sRlJn1Xi/27D8REY/3YDlm1kc+ZzcrRNNkD+DvJH1D0rqZ3iBpnaSNkjYepNn5oZl1r+lh/FkRsUXSScCNkr4TEbe0vyEi1gPrAY7TsiOvlz6zo0SjPXtEbKn/bgeuB9b0Iigz672uk13SIkmLDz8H3grc06vAzKy3mhzGLweuV1XfOA/4fET8bU+iKsy8FScny3NtzlN1xrsvODM572kX358s37F/YbL88atfmiw/6Yudl5+9/yDTN3srU0+f65e+kSOw3/iukz0iHgL+TQ9jMbM+ctWbWSGc7GaFcLKbFcLJblYIJ7tZIdzEdQS0dia6W4ZsNY8m5ncs23la+vf8PctvTZa/dWF62OTv/Y/dyfJ1D/6XjmUT96abiU49/kSyfJjVX2OLFiXLWw2G0e4X79nNCuFkNyuEk92sEE52s0I42c0K4WQ3K4ST3awQrmcfgPHjlyTLp556Olk+NjmZLG/t29ex7JSbdybn/fLPrEqWv3Xh7cnyyUyPyot/fXPHsj1vytSjNxSHOt8joHmZr35myOZRrEfP8Z7drBBOdrNCONnNCuFkNyuEk92sEE52s0I42c0K4Xr2AcjVo+eMnXxSsry16V87lsXGdFf+f/31dFfTH/7pm5Ply8ePSZa/++TbOpb9yaIzkvMylt4XtXZ13w9AdkjmVqatfG7I5hHsatp7drNCONnNCuFkNyuEk92sEE52s0I42c0K4WQ3K4Tr2Y8ArSee7NuyJx9r9ns/oXS778Vjz3Qsy7UJH1u8OF2+MD2cdGvv3mR5euapdHmunn0EZf/Tkq6QtF3SPW3Tlkm6UdID9d+l/Q3TzJqazc/6BuCcadM+AtwUEacDN9WvzWyEZZM9Im4BdkybfB5wZf38SuD8HsdlZj3W7Tn78ojYWj9/FFje6Y2S1gHrACZJn2OZWf80vhofEQF0vOs/ItZHxOqIWD3BgqarM7MudZvs2yStAKj/bu9dSGbWD90m+w3ARfXzi4Av9SYcM+uX7Dm7pGuAtcCJkjYDHwU+Blwn6b3Aw8AF/QzySJcaPx0gDh5IlmfbbSfqfMczddWHFqbbXb9k3rHJ8r2tdOyTDU4UW3vS9eSaaHCbyFj6/oBsPfsRKLu1IuLCDkVn9zgWM+sj3y5rVggnu1khnOxmhXCymxXCyW5WCDdxHYBc1VrTqrlUU884kJ53Yle6qebBaFYF9aqJRDPWXPVXtNLF+/d3EVGtYdXa2IL03aCpYbSHxXt2s0I42c0K4WQ3K4ST3awQTnazQjjZzQrhZDcrhOvZByE7vG+6Pjkn1yVzijIjC4+Rjn3hWPoegd9+7PWdlz2Zqat+pnM31P2meenUGMV69Bzv2c0K4WQ3K4ST3awQTnazQjjZzQrhZDcrhJPdrBCuZx8AzZtIlufaq/fTgePSFe3jarY/uOqWszqWnb73tkbLzhk/fknHsqmnnk7Om6tnj0OHuoppmLxnNyuEk92sEE52s0I42c0K4WQ3K4ST3awQTnazQriefQCy9eiZ9u7jy5Ymy6ee2DHXkJ41djC97q2HdifLF4+lv0ILt3TuG77fddmtPd23h9f8dDt9jsb27JKukLRd0j1t0y6TtEXSnfXj3P6GaWZNzeYwfgNwzgzTPxERq+rHV3oblpn1WjbZI+IWoPvjRDMbCU0u0H1A0l31YX7Hk0pJ6yRtlLTxIA3G5jKzRrpN9k8DrwBWAVuByzu9MSLWR8TqiFg9QbqDQTPrn66SPSK2RcRURLSAPwbW9DYsM+u1rpJd0oq2l+8A7un0XjMbDdl6dknXAGuBEyVtBj4KrJW0CghgE/C+PsZ49ItM5+1T3Y8lrtw44hPpdS/J9AvfIt3n/cHFieU3bCufradP3N+gifTnmtq5s6uYRlk22SPiwhkmf7YPsZhZH/l2WbNCONnNCuFkNyuEk92sEE52s0K4iesgjHVu5glAK121FgcOdr3q2J++RXn8mXQT11zV2gKlu8le8kDnsmzT38x2yzVDTTWR1UT31XZwZHY17T27WSGc7GaFcLKbFcLJblYIJ7tZIZzsZoVwspsVwvXsgxDpuuqc1jOZLpFT9dGZOvycY8cmk+W7W+kulU/8x20dy5pFBnGg+6Guo0GzYYBoZZoljyDv2c0K4WQ3K4ST3awQTnazQjjZzQrhZDcrhJPdrBCuZx+EXFfRmSGbc/NrPDF/ZkjlyDS1z/nq3pOS5VMPPNRsBQmN2owfgfXkTXnPblYIJ7tZIZzsZoVwspsVwsluVggnu1khnOxmhZjNkM2nAp8DllMN0bw+Ij4paRnw58BKqmGbL4iIJ/sX6tFL4+nK7lx9crJ/9Ez/5gsy/7GnW+m29J/83juS5cfwvY5ludhyQzpHpq1+avman+7vPttvfK7f+f1NW+v33mz27IeASyLiNcCZwPslvQb4CHBTRJwO3FS/NrMRlU32iNgaEXfUz3cB9wGnAOcBV9ZvuxI4v19Bmllzczpnl7QSeD1wG7A8IrbWRY9SHeab2YiadbJLOhb4S+CDEbGzvSwigup8fqb51knaKGnjQdLjjplZ/8wq2SVNUCX61RHxhXryNkkr6vIVwPaZ5o2I9RGxOiJWT7CgFzGbWReyyS5JwGeB+yLi99qKbgAuqp9fBHyp9+GZWa/MponrjwI/B9wt6c562qXAx4DrJL0XeBi4oD8hHv2y3RI3GPI5V2134t3prqC/deCYZPmW7ccny1+ZiD035HK2C+2M5GfPVOtll92gG+thySZ7RNwKdGowfXZvwzGzfvEddGaFcLKbFcLJblYIJ7tZIZzsZoVwspsVwl1JD4AWpO8cjP3p24jHFi5Mlrf27p1zTIdN3H5/svw4pWM746XfT5Y/k7gHIFuPnuuCu4FcE9am3XuPIu/ZzQrhZDcrhJPdrBBOdrNCONnNCuFkNyuEk92sEK5nH4BsV9G5+XNdLufauye09uxJlq/K3CPwm6d9MVn+y7yxc2HDumpNpNvDZ+vSU8vOtLXP3RsxirxnNyuEk92sEE52s0I42c0K4WQ3K4ST3awQTnazQriefQBy7c3Hly5Nlk892WAk7Ey77LHJyWT5629/Z7L8mz987ZxD6pUm9ejZZR+B9eg53rObFcLJblYIJ7tZIZzsZoVwspsVwsluVggnu1khsvXskk4FPgcsp2p6vT4iPinpMuAXgcfqt14aEV/pV6BHs0b16Bm5tvStfenx2fnqsmTxy5/+D8ny07kjvXwbmNncVHMIuCQi7pC0GPiGpBvrsk9ExO/2Lzwz65VsskfEVmBr/XyXpPuAU/odmJn11pzO2SWtBF4P3FZP+oCkuyRdIWnGez4lrZO0UdLGgxx9tyCaHSlmneySjgX+EvhgROwEPg28AlhFtee/fKb5ImJ9RKyOiNUTpPszM7P+mVWyS5qgSvSrI+ILABGxLSKmIqIF/DGwpn9hmllT2WSXJOCzwH0R8Xtt01e0ve0dwD29D8/MemU2V+N/FPg54G5Jd9bTLgUulLSKqjpuE/C+vkR4FBhbtChZnmtO2WTI52g16675pD/8p2T5ir94UbK884DNNmizuRp/KzBTo2jXqZsdQXwHnVkhnOxmhXCymxXCyW5WCCe7WSGc7GaFcFfSA5DrSjo3dHFuyOY4dKjreTU/3ZV0rgns1GOPJcttdHjPblYIJ7tZIZzsZoVwspsVwsluVggnu1khnOxmhVBk6nh7ujLpMeDhtkknAo8PLIC5GdXYRjUucGzd6mVsp0XEjJ0MDDTZX7ByaWNErB5aAAmjGtuoxgWOrVuDis2H8WaFcLKbFWLYyb5+yOtPGdXYRjUucGzdGkhsQz1nN7PBGfae3cwGxMluVoihJLukcyTdL+lBSR8ZRgydSNok6W5Jd0raOORYrpC0XdI9bdOWSbpR0gP13xnH2BtSbJdJ2lJvuzslnTuk2E6VdLOkb0u6V9LF9fShbrtEXAPZbgM/Z5c0DvwL8BZgM3A7cGFEfHuggXQgaROwOiKGfgOGpDcBu4HPRcQP1tM+DuyIiI/VP5RLI+K/jUhslwG7hz2Mdz1a0Yr2YcaB84H3MMRtl4jrAgaw3YaxZ18DPBgRD0XEAeBa4LwhxDHyIuIWYMe0yecBV9bPr6T6sgxch9hGQkRsjYg76ue7gMPDjA912yXiGohhJPspwCNtrzczWuO9B/B3kr4had2wg5nB8ojYWj9/FFg+zGBmkB3Ge5CmDTM+Mtuum+HPm/IFuhc6KyLeALwNeH99uDqSojoHG6W601kN4z0oMwwz/qxhbrtuhz9vahjJvgU4te31S+ppIyEittR/twPXM3pDUW87PIJu/Xf7kON51igN4z3TMOOMwLYb5vDnw0j224HTJb1M0nzgncANQ4jjBSQtqi+cIGkR8FZGbyjqG4CL6ucXAV8aYizPMyrDeHcaZpwhb7uhD38eEQN/AOdSXZH/LvCrw4ihQ1wvB75VP+4ddmzANVSHdQeprm28FzgBuAl4APh7YNkIxXYVcDdwF1VirRhSbGdRHaLfBdxZP84d9rZLxDWQ7ebbZc0K4Qt0ZoVwspsVwsluVggnu1khnOxmhXCymxXCyW5WiP8P9Uwhmc6b90UAAAAASUVORK5CYII=\n", | |
| "text/plain": [ | |
| "<Figure size 432x288 with 1 Axes>" | |
| ] | |
| }, | |
| "metadata": { | |
| "needs_background": "light" | |
| } | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "pr8k-z7IQZPu" | |
| }, | |
| "source": [ | |
| "# Results\n", | |
| "\n", | |
| "| Method | Source | Batch size | Time* |\n", | |
| "| ------------------------ |---------- | ---------- | ------ |\n", | |
| "| `.flow()` | np.array | 1 | 584 ms |\n", | |
| "| | | 8 | 293 ms |\n", | |
| "| | | 32 | 285 ms |\n", | |
| "| | | 256 | 280 ms |\n", | |
| "| `flow_from_directory()` | files | 1 | 3.96 s |\n", | |
| "| | | 8 | 3.50 s |\n", | |
| "| | | 32 | 3.39 s |\n", | |
| "| | | 256 | 3.41 s |\n", | |
| "| `flow_from_dataframe()` | files | 1 | 3.97 s |\n", | |
| "| | | 8 | 3.50 s |\n", | |
| "| | | 32 | 3.41 s |\n", | |
| "| | | 256 | 3.39 s |\n", | |
| "| `.flow()` | memmap | 1 | 574 ms |\n", | |
| "| | | 8 | 296 ms |\n", | |
| "| | | 32 | 278 ms |\n", | |
| "| | | 256 | 274 ms |\n", | |
| "* time to generate 2 epochs (i.e. 2x the whole testing set)" | |
| ] | |
| } | |
| ] | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment