Numpy Neural Net

Luke_Floden_Problem_3.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Jan 25 19:04:21 2020

@author: bionboy

references:
    https://towardsdatascience.com/lets-code-a-neural-network-in-plain-numpy-ae7e74410795

Consider the neural network:
    All the activation functions from the neurons in the hidden layer are
    sigmoids and the error is calculated by using squared error function:
        a.Describe all the essential parts from the neural network.
        b.Given the input {1,1} compute the predicted output of the network 
        step by step and calculate the error if the target output is 0.
        c.Compute step by step 2 training epochs using Back-Propagation algorithm

(1) (3)
    (4) (6)
(2) (5)

1-3: .8
1-4: .4
1-5: .3

2-3: .2
2-4: .9
2-5: .5

3-6: .3
4-6: .5
5-6: .9
"""

import numpy as np
import matplotlib.pyplot as plt

DEBUG = True
DEEP_DEBUG = False

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_backward(d_a, x):
    sig = sigmoid(x)
    return d_a * sig * (1 - sig)

def sqr_error(y, truth):
    return np.power((y - truth), 2)

def forward_propagate_layer(prev, curr):
    if (DEEP_DEBUG): print(f'propagate:\n\tprev: {prev.shape}\n\tcurr: {curr.shape}')

    z = np.dot(curr.T, prev)
    a = sigmoid(z)
    return a

def forward_propagate(input_data, weights):
    if (DEBUG): print(f'--> {input_data}')
    a_curr = input_data
    for layer in weights:
        a_prev = a_curr
        w_curr = np.array(layer)
        a_curr = forward_propagate_layer(a_prev, w_curr)
        if (DEBUG): print(f'--> {a_curr}')
    return a_curr


input_data = np.array([1,1])
weights = np.array([[[.8, .4, .3], [.2, .9, .5]],
                    [[.3], [.5], [.9]]])

output = forward_propagate(input_data, weights)
error = sqr_error(output, 0)

print(f'(a)\tinput:\t{input_data}\n\toutput:\t{output}\n\terror:\t{error}')


'''
def single_layer_backward_propagation(dA_curr, W_curr, b_curr, Z_curr, A_prev):
    m = A_prev.shape[1]
    
    dZ_curr = sigmoid(dA_curr, Z_curr)
    dW_curr = np.dot(dZ_curr, A_prev.T) / m
    db_curr = np.sum(dZ_curr, axis=1, keepdims=True) / m
    dA_prev = np.dot(W_curr.T, dZ_curr)

    return dA_prev, dW_curr, db_curr

def full_backward_propagation(Y_hat, Y, memory, params_values, nn_architecture):
    grads_values = {}
    m = Y.shape[1]
    Y = Y.reshape(Y_hat.shape)

    dA_prev = - (np.divide(Y, Y_hat) - np.divide(1 - Y, 1 - Y_hat));
    
    for layer_idx_prev, layer in reversed(list(enumerate(nn_architecture))):
        layer_idx_curr = layer_idx_prev + 1
        activ_function_curr = layer["activation"]
        
        dA_curr = dA_prev
        
        A_prev = memory["A" + str(layer_idx_prev)]
        Z_curr = memory["Z" + str(layer_idx_curr)]
        W_curr = params_values["W" + str(layer_idx_curr)]
        b_curr = params_values["b" + str(layer_idx_curr)]
        
        dA_prev, dW_curr, db_curr = single_layer_backward_propagation(
            dA_curr, W_curr, b_curr, Z_curr, A_prev, activ_function_curr)
        
        grads_values["dW" + str(layer_idx_curr)] = dW_curr
        grads_values["db" + str(layer_idx_curr)] = db_curr
    
    return grads_values

'''

def backward_propagation(d_a_curr, w_curr, z_curr, a_prev):
    m = a_prev.shape[1]

    d_z_curr = sigmoid_backward(d_a_curr, z_curr)
    d_w_curr = np.dot(d_z_curr, a_prev.T) / m
    d_a_prev = np.dot(w_curr.T, d_z_curr)

    return d_a_prev, d_w_curr