MLP mnist numpy code

layers.py

import numpy as np

class Layer:
    every = []
    def __init__(self):
        self._result = None
        Layer.every.append(self)

    @classmethod
    def clean_all(cls):
        for e in cls.every:
            e.clear()

    def clear(self):
        self._result = None

    def forward(self):
        raise "not implemented!"

    def backward(self):
        raise "not implemented!"

    def value(self):
        if self._result is None:
            self._result = self.forward()
        return self._result


class MMul(Layer):
    def __init__(self, A, B):
        super().__init__()
        self.A = A
        self.B = B
    
    def forward(self):
        return self.A.value() @ self.B.value()

    def backward(self, loss):  
        A = self.A.value()
        B = self.B.value()
        self.A.backward(loss @ B.T)
        self.B.backward(A.T @ loss)


class Variable(Layer):
    def __init__(self, initval):
        super().__init__()
        self.val = initval
        self.normalizer = np.zeros(self.val.shape)

    def forward(self):
        return self.val

    def backward(self, loss):
        grd = np.average(loss, 0)
        self.normalizer = 0.9*self.normalizer + 0.1*(grd**2)
        self.val -= 1e-3*grd/((self.normalizer + 1e-10)**0.5)


class Input(Layer):
    def __init__(self):
        super().__init__()
        self.current = None
    
    def set_val(self, inp):
        self.current = inp

    def forward(self):
        return self.current

    def backward(self, loss):
        pass


class Sigmoid(Layer):
    def __init__(self, input):
        super().__init__()
        self.input = input

    def sigm(self, inp):
        return 1.0/(1.0+np.exp(-inp))

    def forward(self):
        return self.sigm(self.input.value())

    def backward(self, loss):
        self.input.backward(self.value() * (1-self.value()) * loss)


class LogLoss(Layer):
    def __init__(self, pred, tgt):
        super().__init__()
        self.py = pred
        self.y = tgt

    def forward(self):
        y = self.y.value()
        py = self.py.value()
        self.last = (y, py)
        return -np.mean(y * np.log(py) + (1-y) * np.log(1-py), 1)

    def backward(self, loss):
        y = self.y.value()
        py = self.py.value()
        n = py.shape[1]
        self.py.backward((-1./n)*((y/py - (1-y)/(1-py)) * loss))
        # no self.y.backward() ! 


class Softmax(Layer):
    def __init__(self, input):
        super().__init__()
        self.input = input

    def soft(self, inp):
        ex = np.exp(inp)
        return ex/np.sum(ex, 1, keepdims=True)

    def forward(self):
        return self.soft(self.input.value())

    def backward(self, loss):
        self.input.backward(self.value() * (loss-np.sum(self.value() * loss, 1, keepdims=True)))


class Relu(Layer):
    def __init__(self, input):
        super().__init__()
        self.input = input

    def forward(self):
        return np.maximum(0, self.input.value())

    def backward(self, loss):
        self.input.backward(np.where(self.value() > 0, loss, 0))


class Sum(Layer):
    def __init__(self, A, B):
        super().__init__()
        self.A = A
        self.B = B
    
    def forward(self):
        return self.A.value() + self.B.value()

    def backward(self, loss):   
        self.A.backward(loss)
        self.B.backward(loss)



def fc(inp, ins, outs):
    return Sum(
        MMul(inp, Variable(np.random.randn(ins, outs)/((ins/2)**0.5))),
        Variable(np.random.randn(outs))
    )

run.py

#!/usr/bin/env python3

import numpy as np
from layers import *
import random
from tensorflow.examples.tutorials.mnist import input_data


mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)

inp = Input()
tgt = Input()

dl = Relu(fc(inp, 784, 60))
dl = Relu(fc(dl, 60, 60))
dl = Softmax(fc(dl, 60, 10))
loss = LogLoss(dl, tgt)


train_data = mnist.train
test_data = mnist.test
batch_size = 100

reses = []
for i in range(10000):
    x, y = train_data.next_batch(batch_size)
    Layer.clean_all()
    inp.set_val(x)
    tgt.set_val(y)
    result = loss.value()
    loss.backward(1)
    reses.append(np.average(result))
    if i%1000 == 0:
        print(i, sum(reses)/len(reses))
        reses = []


examples, correct = 0, 0
losses = []
for i in range(test_data.num_examples//batch_size):
    x, y = test_data.next_batch(batch_size)
    Layer.clean_all()
    inp.set_val(x)
    tgt.set_val(y)
    result = dl.value()
    losses.append(loss.value())
    examples += y.shape[0]
    correct += np.sum(np.argmax(result, 1) == np.argmax(y, 1))

print("top1: {}".format(correct/examples))
print("loss: {}".format(np.average(np.asarray(losses))))