two_layer_ann.py

import random, math
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patheffects as mpe


"""
dataset
"""

xin = [
  [0,0,1,1],
  [0,1,0,1],
]

target=[1,1,1,0]

"""
helper funtions
"""

def display_mat(mat):
  for r in range(len(mat)):
    for c in range(len(mat[0])):
      print(f"{mat[r][c]:.3f}", end=" "*3)
    print("\n")

def get_mat(n_rows, n_cols, urange=(-2.0,2.0)):
  return [
      [random.uniform(*urange) 
        for i in range(n_cols)] 
            for j in range(n_rows)
    ]

pe = [mpe.Stroke(linewidth=8, foreground='black'), mpe.Stroke(foreground='white',alpha=1), mpe.Normal()]

N_SAMPLES = 4
INPUT_SIZE = 2

UNFRM_INIT = (-2.0, 2.0)
N_HID_SIZE = 4
N_OUT_SIZE = 1

LR = 0.01
STOP_AT_LOSS = 0.1

PRED_THRESH = 0.5


W1  = get_mat(n_rows=INPUT_SIZE, n_cols=N_HID_SIZE, urange=UNFRM_INIT)
W2  = get_mat(n_rows=N_OUT_SIZE, n_cols=N_HID_SIZE, urange=UNFRM_INIT)

B1 = [random.uniform(*UNFRM_INIT) for i in range(N_HID_SIZE)]
B2 = random.uniform(*UNFRM_INIT)


def sigmoid(x):
    return 1/(1+np.exp(-x))


while True: 
    
    sample_preds = [0 for i in range(N_SAMPLES)]
    for i in range(0, N_SAMPLES):
        
        # fwd prop (layer 1)
        z1= (xin[0][i]*W1[0][0]) + (xin[1][i]*W1[1][0]) + B1[0]
        z2= (xin[0][i]*W1[0][1]) + (xin[1][i]*W1[1][1]) + B1[1]
        z3= (xin[0][i]*W1[0][2]) + (xin[1][i]*W1[1][2]) + B1[2]
        z4= (xin[0][i]*W1[0][3]) + (xin[1][i]*W1[1][3]) + B1[3]

        o1 = sigmoid(z1)
        o2 = sigmoid(z2)
        o3 = sigmoid(z3)
        o4 = sigmoid(z4)

        # fwd prop (layer 2)
        h1_out = [o1,o2,o3,o4]
        u = (h1_out[0]*W2[0][0]) + (h1_out[1]*W2[0][1]) + (h1_out[2]*W2[0][2])+ (h1_out[3]*W2[0][3]) + B2
        
        o4 = sigmoid(u)
        sample_preds[i] = o4
        
        # bck prop (layer 2)
        dW2 = [
            LR*h1_out[0]*(target[i]-o4)*o4*(1-o4),
            LR*h1_out[1]*(target[i]-o4)*o4*(1-o4)
        ]

        db2 = LR*(target[i]-o4)*o4*(1-o4)

        W2[0][0] = W2[0][0] + dW2[0]
        W2[0][1] = W2[0][1] + dW2[1]
        B2 = B2 + db2
        
        # bck prop (layer 1)
        dW1 = get_mat(n_rows=INPUT_SIZE, n_cols=N_HID_SIZE)
        sum=0
        sum= (W2[0][0]*o4*(1-o4)*(target[i]-o4)) + (W2[0][1]*o4*(1-o4)*(target[i]-o4)) + (W2[0][2]*o4*(1-o4)*(target[i]-o4))

        for ij in range(0,2):
            for jk in range(0,2):
                dW1[ij][jk]= LR*xin[ij][i]*h1_out[jk]*(1-h1_out[jk])*sum        

        dB1 = LR*h1_out[0]*(1-h1_out[0])*sum
        dB2 = LR*h1_out[1]*(1-h1_out[1])*sum
        dB3 = LR*h1_out[2]*(1-h1_out[2])*sum 

        for ij in range(INPUT_SIZE):
            for jk in range(N_HID_SIZE):
                W1[ij][jk] = W1[ij][jk] + dW1[ij][jk] 

        B1[0]= B1[0] + dB1
        B1[1]= B1[1] + dB2
        B1[2]= B1[2] + dB3 


    # At the end of epoh, calc errors
    errors = [0 for i in range(0, N_SAMPLES)]
    for e in range(0, N_SAMPLES):
        errors[e] = math.pow((target[e] - sample_preds[e]), 2)
    
    print(np.mean(errors))
    if(np.mean(errors)<STOP_AT_LOSS):
        break


"""
predcition
"""

print("\n\nPrediction after training:")

sample_preds = [0 for i in range(N_SAMPLES)]
for i in range(0, N_SAMPLES):
    
    # fwd prop (layer 1)
    z1= (xin[0][i]*W1[0][0]) + (xin[1][i]*W1[1][0]) + B1[0]
    z2= (xin[0][i]*W1[0][1]) + (xin[1][i]*W1[1][1]) + B1[1]
    z3= (xin[0][i]*W1[0][2]) + (xin[1][i]*W1[1][2]) + B1[2]
    z4= (xin[0][i]*W1[0][3]) + (xin[1][i]*W1[1][3]) + B1[3]

    o1 = sigmoid(z1)
    o2 = sigmoid(z2)
    o3 = sigmoid(z3)
    o4 = sigmoid(z4)

    # fwd prop (layer 2)
    h1_out = [o1,o2,o3,o4]
    u = (h1_out[0]*W2[0][0]) + (h1_out[1]*W2[0][1]) + (h1_out[2]*W2[0][2])+ (h1_out[3]*W2[0][3]) + B2
    
    o4 = sigmoid(u)
    sample_preds[i] = o4

sample_preds = [int(pred>PRED_THRESH) for pred in sample_preds]
print('predicted:', sample_preds)
print('target:', target, end="\n\n")


"""
plot results
"""

fig = plt.figure(figsize=(5,5))

for x1 in np.linspace(-1, 2, 100):
    for x2 in np.linspace(-1, 2, 100):

        z1= (x1*W1[0][0]) + (x2*W1[1][0]) + B1[0]
        z2= (x1*W1[0][1]) + (x2*W1[1][1]) + B1[1]
        z3= (x1*W1[0][2]) + (x2*W1[1][2]) + B1[2]
        z4= (x1*W1[0][3]) + (x2*W1[1][3]) + B1[3]

        o1 = sigmoid(z1)
        o2 = sigmoid(z2)
        o3 = sigmoid(z3)
        o4 = sigmoid(z4)

        h1_out = [o1,o2,o3,o4]
        u = (h1_out[0]*W2[0][0]) + (h1_out[1]*W2[0][1]) + (h1_out[2]*W2[0][2])+ (h1_out[3]*W2[0][3]) + B2
        
        o4 = sigmoid(u)

        color = (0,0,o4,0.2) if o4<PRED_THRESH else (o4,0,0,0.2)
        plt.plot([x1], [x2], color=color, marker='o')

for x1, x2, t in zip(xin[0], xin[1], target):
    color = 'blue' if t==0 else 'red'
    plt.plot([x1], [x2], color=color, marker='o', linewidth=1, markersize=10, path_effects=pe)

plt.show()