study.ipynb

mnist-scratch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.6"
    },
    "colab": {
      "name": "study.ipynb",
      "private_outputs": true,
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/klein-mask/19a56004a05f00e5d6de75c8b02efd16/study.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0-EB3K0JbF-j"
      },
      "source": [
        "# This notebook is my learnig for JDLA.\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iSuf8mdLbF-n"
      },
      "source": [
        "## Study MNIST dataset by full scratch code."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WBZQy5I8fYOC"
      },
      "source": [
        "### 1. Load dataset from sklearn ."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kmeuCfK6bF-o"
      },
      "source": [
        "from sklearn.datasets import fetch_openml\n",
        "import numpy as np\n",
        "\n",
        "X_train, y_train = fetch_openml('mnist_784', version=1, return_X_y=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o0OzlXn6bF-p"
      },
      "source": [
        "print(X_train.shape, y_train.shape)\n",
        "print(X_train.ndim, y_train.ndim)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vFKtuv4FfjES"
      },
      "source": [
        "### 2. Normalization by divide 255 ."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NS6DDjPXfGax"
      },
      "source": [
        "X_train = X_train.astype(np.float) / 255\n",
        "X_train[0]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hs470lV0gFBd"
      },
      "source": [
        "### 3. Convert labels by one hot encording .\n",
        "\n",
        "\n",
        "**example**\n",
        "\n",
        "'5' → [0, 0, 0, 0, 1, 0, 0, 0, 0]\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZovdaChvgsoG"
      },
      "source": [
        "def to_one_hot(labels: np.array, n_class: int) -> np.array:\n",
        "    if labels.ndim > 1 and labels.shape[1] == n_class:\n",
        "        return labels\n",
        "\n",
        "    one_hot_labels = np.zeros((labels.shape[0], n_class)) # 0 ~ 9\n",
        "\n",
        "    for i, label in enumerate(labels):\n",
        "        one_hot_labels[i, int(label)] = 1\n",
        "    return one_hot_labels"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8S24xHFqii7Q"
      },
      "source": [
        "y_train = to_one_hot(y_train, 10)\n",
        "y_train.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "peirulTSkVlc"
      },
      "source": [
        "### 4. Define activation and loss functions ."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0FUAXtSOkiGz"
      },
      "source": [
        "from abc import ABCMeta, abstractmethod\n",
        "from dataclasses import dataclass, InitVar, field\n",
        "from typing import List\n",
        "\n",
        "class Neuron(metaclass=ABCMeta):\n",
        "    @abstractmethod\n",
        "    def forward(self):\n",
        "        pass\n",
        "\n",
        "    @abstractmethod\n",
        "    def backward(self):\n",
        "        pass"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DZ8HUuBmt5hE"
      },
      "source": [
        "@dataclass\n",
        "class Affine(Neuron):\n",
        "    input_size: np.ndarray\n",
        "    hidden_size: np.ndarray\n",
        "\n",
        "    W: np.ndarray = field(init=False)\n",
        "    b: np.ndarray = field(init=False)\n",
        "    x: np.ndarray = field(init=False)\n",
        "    x_shape: tuple = field(init=False)\n",
        "    learning_rate: float = field(default=0.1)\n",
        "\n",
        "    def __post_init__(self):\n",
        "        self.W = np.random.randn(self.input_size, self.hidden_size)\n",
        "        self.b = np.zeros(self.hidden_size)\n",
        "\n",
        "\n",
        "    def forward(self, x: np.ndarray) -> np.ndarray:\n",
        "        self.x_shape = x.shape\n",
        "        self.x = x.reshape(x.shape[0], -1)\n",
        "        out = np.dot(self.x, self.W) + self.b\n",
        "        return out\n",
        "\n",
        "    def backward(self, dout: np.ndarray) -> np.ndarray:\n",
        "        dx = np.dot(dout, self.W.T).reshape(*self.x_shape)\n",
        "\n",
        "        dW = np.dot(self.x.T, dout)\n",
        "        db = np.sum(dout, axis=0)\n",
        "\n",
        "        self.W -= self.learning_rate * dW\n",
        "        self.b -= self.learning_rate * db\n",
        "\n",
        "        return dx"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8c-tikoxt_H7"
      },
      "source": [
        "@dataclass\n",
        "class ReLU(Neuron):\n",
        "    mask: bool = field(init=False)\n",
        "\n",
        "    def forward(self, x: np.ndarray) -> np.ndarray:\n",
        "        self.mask = (x <= 0)\n",
        "        out = x.copy()\n",
        "        out[self.mask] = 0\n",
        "        return out\n",
        "\n",
        "    def backward(self, dout: np.ndarray) -> np.ndarray:\n",
        "        dout[self.mask] = 0\n",
        "        dx = dout\n",
        "        return dx"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0qSmTgBruHL4"
      },
      "source": [
        "@dataclass\n",
        "class SoftmaxWithLoss(Neuron): # output layer\n",
        "    true_y: np.ndarray = field(init=False)\n",
        "    pred_y: np.ndarray = field(init=False)\n",
        "    loss: np.ndarray = field(init=False)\n",
        "\n",
        "    def forward(self, x: np.ndarray, t: np.ndarray) -> np.ndarray:\n",
        "        self.true_y = t\n",
        "        self.pred_y = self.softmax(x)\n",
        "        self.loss = self.cross_entropy_error(y=self.pred_y, t=self.true_y)\n",
        "        return self.loss\n",
        "\n",
        "    def backward(self) -> np.ndarray:\n",
        "        batch_size = self.pred_y.shape[0]\n",
        "        dx = (self.pred_y - self.true_y) / batch_size\n",
        "        return dx\n",
        "    \n",
        "    def softmax(self, x: np.ndarray) -> np.ndarray:\n",
        "        x = x - np.max(x, axis=-1, keepdims=True) # xが大きすぎるとオーバーフローするため\n",
        "        return np.exp(x) / np.sum(np.exp(x), axis=-1, keepdims=True)\n",
        "    \n",
        "    def cross_entropy_error(self, y: np.ndarray, t: np.ndarray):\n",
        "        label_idxs  = t.argmax(axis=1)\n",
        "        batch_size = y.shape[0]\n",
        "\n",
        "        pred_y_labels = y[np.arange(batch_size), label_idxs]\n",
        "        return -np.sum(np.log(pred_y_labels + 1e-7)) / batch_size # 0除算にならないよう微小な値を加える"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0-MhNt_1tg7h"
      },
      "source": [
        "@dataclass\n",
        "class Network:\n",
        "    layers: List[Neuron]\n",
        "    last_layer: Neuron\n",
        "\n",
        "    def predict(self, x: np.ndarray) -> np.ndarray:\n",
        "        for layer in self.layers:\n",
        "            x = layer.forward(x)\n",
        "        return x\n",
        "\n",
        "    def loss(self, x: np.ndarray, t: np.ndarray) -> float:\n",
        "        x = self.predict(x)\n",
        "        return self.last_layer.forward(x=x, t=t)\n",
        "\n",
        "    def fit(self):\n",
        "        dout = self.last_layer.backward()\n",
        "\n",
        "        for layer in reversed(list(self.layers)):\n",
        "            dout = layer.backward(dout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TqRBOmcQtJO6"
      },
      "source": [
        "### 5. Create network."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VftB2rxXtTYW"
      },
      "source": [
        "np.random.seed(0)\n",
        "\n",
        "lr = 0.025\n",
        "\n",
        "network = Network(\n",
        "    layers=[\n",
        "        Affine(input_size=784, hidden_size=32, learning_rate=lr),\n",
        "        ReLU(),\n",
        "        Affine(input_size=32, hidden_size=32, learning_rate=lr),\n",
        "        ReLU(),\n",
        "        Affine(input_size=32, hidden_size=10, learning_rate=lr)\n",
        "    ],\n",
        "    last_layer=SoftmaxWithLoss()\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GkpbunMqyCbi"
      },
      "source": [
        "### 6. Study network model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5gs9CDzsvvSb"
      },
      "source": [
        "def fit(batch_size, n_epochs):\n",
        "    for i in range(n_epochs):\n",
        "        batch_indexes = np.random.choice(X_train.shape[0], batch_size, replace=False)\n",
        "        \n",
        "        X_batch = X_train[batch_indexes]\n",
        "        y_batch = y_train[batch_indexes]\n",
        "\n",
        "        e = network.loss(x=X_batch, t=y_batch)\n",
        "        network.fit()\n",
        "\n",
        "        if i % batch_size == 0:\n",
        "            print(f'loss: {e}')\n",
        "    return network"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pmT8zaDVijLt"
      },
      "source": [
        "fitted_network = fit(batch_size=128, n_epochs=20000)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kjdmG9o9M-EZ"
      },
      "source": [
        "### 7. Predict mnist data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zxzveUr-NDoz"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "def show_mnist(x):\n",
        "    x = x.reshape(28, 28)\n",
        "    plt.imshow(x)\n",
        "    plt.gray()\n",
        "    plt.show()\n",
        "\n",
        "for index in range(10):\n",
        "    show_mnist(X_train[index])\n",
        "\n",
        "    true_label = np.argmax(y_train[index])\n",
        "    pred_label = np.argmax(network.predict(np.array([X_train[index]]))[0])\n",
        "\n",
        "    print(f'true_label: {true_label}')\n",
        "    print(f'pred_label: {pred_label}')\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pOp9tbL4M8_3"
      },
      "source": [
        "\n"
      ]
    }
  ]
}