Short implementation of a feedforward neural network with backpropagation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import numpy as np | |
import math | |
from pprint import pprint | |
import pdb | |
#Linearly separable | |
_or = { | |
'X': [[0,0],[0,1],[1,0],[1,1]], | |
'y': [0, 1, 1, 1] | |
} | |
#Linearly inseparable | |
xor = { | |
'X': [[0,0],[0,1],[1,0],[1,1]], | |
'y': [0,1,1,0] | |
} | |
def activate(X): | |
return math.tanh(X) | |
def derivative_activate(X): | |
return 1.0 - X**2 | |
def addBiasColumn(listoflists, bias_value=1): | |
#Append the bias value to every row in listoflists | |
return [ [bias_value] + i for i in listoflists] | |
#Create array or matrix of completely random numbers | |
def createRandomVector(n_items):#n_items can be number or tuple | |
return np.random.random(n_items) | |
class neuralNetwork: | |
def __init__(self): | |
self.learning_rate = 0.1 | |
def feedforward(self, input): | |
#Step 1: Lets sum the weights * input of each hidden node | |
for i in range(len(self.hidden_layer_nodes )): | |
#Vectorized implementation of ( sum of weights attached to hidden node * input ) | |
hidden_node_sum = sum( input * self.input_weights[i] ) | |
#Lets apply the activation function - specifically the sigmoid | |
self.hidden_layer_nodes[i] = activate( hidden_node_sum ) | |
#print i, " " ,input, self.input_weights[i], input * self.input_weights[i] | |
#Step 2: Lets sum the weights * hidden node values to get our output | |
total, guess = 0, 0 | |
#Go through all the hidden nodes and multiply their values by their weights | |
for i in range(len(self.hidden_layer_nodes )): | |
total += self.hidden_layer_nodes[i] * self.hidden_layer_weights[i] | |
guess = activate( total )#Sigmoid of the total output of the neural network | |
return guess | |
def fit(self, X, y, num_iterations=1000): | |
#Return if the size of all inputs in X do not match | |
if not all( len(x)==len(X[0]) for x in X): | |
raise Exception( "Not all elements in input X have the same length" ) | |
#Lets add a bias column and turn the data into numpy arrays for easier matrix operations | |
self.X = X = np.array( addBiasColumn(X) ) | |
self.y = y = np.array( map(float,y) ) | |
single_input_size = len( X[0] ) | |
#Lets create an n x 1 column vector of hidden layer nodes | |
self.hidden_layer_nodes = createRandomVector( single_input_size ) | |
num_hidden_nodes = len( self.hidden_layer_nodes ) | |
#Lets create a matrix of weights for input and hidden nodes | |
self.input_weights = createRandomVector( (num_hidden_nodes, single_input_size) ) | |
self.hidden_layer_weights = createRandomVector( num_hidden_nodes ) | |
for i in range(num_iterations): | |
#Lets cycle through each input(X) and the corresponding answer (y) | |
for input,output in zip(X, y): | |
#Feedforward function gives us the 'guess' for the algorithm | |
guess = self.feedforward(input) | |
error = output - guess#How far off were we? | |
#Now lets take our guess and error and improve our weights for the system | |
self.backPropagation(input, guess, error) | |
if i % 100 == 0: | |
print .5*error**2 | |
#Lets test the input against our weights to see how we did | |
def predict(self, input=None): | |
#If we are using our orginal input | |
if input is None: | |
inputs = xor['X'] | |
for input in inputs: | |
input_with_bias = [1] + input | |
print "Input %s = %s" % ( str(input), self.feedforward( input_with_bias ) ) | |
#If we are testing out another input lets bias it and then run the program | |
else: | |
biased_input = addBiasColumn(input) | |
for bias in biased_input: | |
print "Input %s = %s" % ( str(bias), self.feedforward( bias ) ) | |
self.weights() | |
#Lets print out what our weights converged to | |
def weights(self): | |
print "Input Weights :\n",self.input_weights | |
print "Hidden layer weights :\n",self.hidden_layer_weights | |
def backPropagation(self, input, guess, error): | |
delta_hidden_nodes = [0]*len(self.hidden_layer_nodes) | |
#Step 1: Lets get the delta of the output node | |
delta_output = error * derivative_activate(guess) | |
#Step 2: Adjust weights for hidden layer -> output | |
for i in range(len(self.hidden_layer_weights)):#Not vectorizing this so its a little clearer | |
self.hidden_layer_weights[i] += self.learning_rate*delta_output*self.hidden_layer_nodes[i] | |
#Step 3: Lets get the hidden layer errors | |
for i in range(len(self.hidden_layer_nodes)): | |
out = self.hidden_layer_nodes[i] | |
delta_hidden_nodes[i] = derivative_activate(out)*( delta_output* self.hidden_layer_weights[i]) | |
#Step 4: Lets change the weights going into the hidden layer | |
row, column = self.input_weights.shape#(num_hidden_nodes, single_input_size) | |
for i in range(row): | |
for j in range(column): | |
self.input_weights[i][j] += self.learning_rate*delta_hidden_nodes[i]*input[j] | |
def run(): | |
X, y = xor['X'], xor['y'] | |
nn = neuralNetwork() | |
nn.fit( X,y)#Lets train the system to get our weights | |
nn.predict(X)#Lets see how we did | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment