2nd attempt at making a nn builder

my_nn_builder_main.py

import neural_network_2

shape = [4, 4, 2]

my_nn = neural_network_2.NeuralNetwork(shape)
print("propagate :", my_nn.propagate([1, 1, 0, 0]))

X_batch = [
    [1, 1, 0, 0],
    [0, 0, 0, 0],
    [1, 0, 1, 0],
    [0, 0, 1, 1],
    [0, 1, 1, 0]]

y_batch = [
    [1, 0],
    [0, 0],
    [1, 0],
    [0, 1],
    [0, 1]]

print("learn :")
my_nn.learn(X_batch, y_batch, learning_rate=0.1)

print("propagate :", my_nn.propagate([1, 1, 0, 0]))

my_nn_builder_my_layer.py

import random


def ReLU(x):
    return max(0, x)


class Layer:
    def __init__(self, input_size, output_size):
        self.shape = (input_size, output_size)
        self.bias = [0 for _ in range(output_size)]
        self.weights = [[random.uniform(-1, 1) for _ in range(input_size)] for _ in range(output_size)]

    def get_z(self, X):
        return [sum([self.weights[i][j] * X[j] for j in range(self.shape[0])]) + self.bias[i]
                for i in range(self.shape[1])]

my_nn_builder_neural_network_2.py

# import random
# import numpy as np

# I decided to make layer object instead of just a list containing bias
import my_layer


def ReLU_on_list(xs):
    return [max(0, x) for x in xs]


def ReLU_derivative(x):
    return 0 if x <= 0 else 1


# first try, without numpy
class NeuralNetwork:
    def __init__(self, shape):

        if len(shape) < 3:
            print("CAUTION : Not enough layers in your network, might crashes")

        self.shape = shape  # something like [4, 4, 2]
        self.layers = [my_layer.Layer(input_s, output_s) for input_s, output_s in zip(self.shape[:-1], self.shape[1:])]

    def propagate(self, X):

        for layer in self.layers:
            X = ReLU_on_list(layer.get_z(X))

        return X

    def learn(self, X_batch, y_batch, learning_rate=0.001):

        len_batch = len(X_batch)
        if len_batch != len(y_batch):
            print("batch size not matching")
            return

        X_sample = X_batch[0]
        y_sample = y_batch[0]

        # as => list of all a
        as_ = [X_sample]

        # zs => list of all z
        zs = []
        activated_z = X_sample

        # we do one full propagation to get all z
        for layer in self.layers:
            z = layer.get_z(activated_z)
            zs.append(z)
            activated_z = ReLU_on_list(z)  # apply activation function
            as_.append(activated_z)

        # list for delta gradient for bias and for weights
        # same shape as the bias and the weights
        dg_b = [[0 for _ in range(i)] for i in self.shape[1:]]
        dg_w = []
        for front_layer_size, back_layer_size in zip(self.shape[:-1], self.shape[1:]):
            dg_w.append([[0 for _ in range(front_layer_size)] for _ in range(back_layer_size)])

        # partial derivative of the 'cost' depending on 'a'
        # same shape as bias
        pd_cas = [[0 for _ in range(i)] for i in self.shape[1:]]

        # back propagation
        # first, the last layer
        # in my notations, j is the index of the neurons of the 'right' layer (next one)
        # and k the index for the 'left' one (previous one)
        for j in range(self.shape[-1]):

            # loss function => loss squared, derivative is 2 * loss
            pd_ca = 2 * (as_[-1][j] - y_sample[j])
            pd_cas[-1][j] = pd_ca

            calculus_body = ReLU_derivative(zs[-1][j]) * pd_ca
            dg_b[-1][j] = calculus_body

            for k in range(self.shape[-2]):
                dg_w[-1][j][k] = as_[-2][k] * calculus_body

        # - 2 because the first layer doesn't count and we already did the last one
        for layer_index in range(-2, -len(self.shape), -1):
            for j in range(self.shape[layer_index]):

                pd_ca = 0
                for sub_j in range(self.shape[layer_index + 1]):
                    w = self.layers[layer_index].weights[sub_j][j]
                    pd_ca += w * ReLU_derivative(zs[layer_index + 1][sub_j] * pd_cas[layer_index + 1][sub_j])

                pd_cas[layer_index][j] = pd_ca

                calculus_body = ReLU_derivative(zs[layer_index][j]) * pd_ca

                dg_b[layer_index][j] = calculus_body

                for k in range(self.shape[layer_index - 1]):

                    dg_w[layer_index][j][k] = as_[layer_index - 1][k] * calculus_body

        # apply delta gradient

        for i in range(len(self.shape) - 1):
            for j in range(self.shape[i + 1]):

                self.layers[i].bias[j] += dg_b[i][j] * learning_rate

                for k in range(self.shape[i]):

                    self.layers[i].weights[j][k] -= dg_w[i][j][k] * learning_rate

        print(dg_b)
        print(dg_w)