Skip to content

Instantly share code, notes, and snippets.

@leonaburime
Last active August 29, 2015 14:03
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save leonaburime/97bf84dffe18bac9da4f to your computer and use it in GitHub Desktop.
Short implementation of a feedforward neural network with backpropagation
import numpy as np
import math
from pprint import pprint
import pdb
#Linearly separable
_or = {
'X': [[0,0],[0,1],[1,0],[1,1]],
'y': [0, 1, 1, 1]
}
#Linearly inseparable
xor = {
'X': [[0,0],[0,1],[1,0],[1,1]],
'y': [0,1,1,0]
}
def activate(X):
return math.tanh(X)
def derivative_activate(X):
return 1.0 - X**2
def addBiasColumn(listoflists, bias_value=1):
#Append the bias value to every row in listoflists
return [ [bias_value] + i for i in listoflists]
#Create array or matrix of completely random numbers
def createRandomVector(n_items):#n_items can be number or tuple
return np.random.random(n_items)
class neuralNetwork:
def __init__(self):
self.learning_rate = 0.1
def feedforward(self, input):
#Step 1: Lets sum the weights * input of each hidden node
for i in range(len(self.hidden_layer_nodes )):
#Vectorized implementation of ( sum of weights attached to hidden node * input )
hidden_node_sum = sum( input * self.input_weights[i] )
#Lets apply the activation function - specifically the sigmoid
self.hidden_layer_nodes[i] = activate( hidden_node_sum )
#print i, " " ,input, self.input_weights[i], input * self.input_weights[i]
#Step 2: Lets sum the weights * hidden node values to get our output
total, guess = 0, 0
#Go through all the hidden nodes and multiply their values by their weights
for i in range(len(self.hidden_layer_nodes )):
total += self.hidden_layer_nodes[i] * self.hidden_layer_weights[i]
guess = activate( total )#Sigmoid of the total output of the neural network
return guess
def fit(self, X, y, num_iterations=1000):
#Return if the size of all inputs in X do not match
if not all( len(x)==len(X[0]) for x in X):
raise Exception( "Not all elements in input X have the same length" )
#Lets add a bias column and turn the data into numpy arrays for easier matrix operations
self.X = X = np.array( addBiasColumn(X) )
self.y = y = np.array( map(float,y) )
single_input_size = len( X[0] )
#Lets create an n x 1 column vector of hidden layer nodes
self.hidden_layer_nodes = createRandomVector( single_input_size )
num_hidden_nodes = len( self.hidden_layer_nodes )
#Lets create a matrix of weights for input and hidden nodes
self.input_weights = createRandomVector( (num_hidden_nodes, single_input_size) )
self.hidden_layer_weights = createRandomVector( num_hidden_nodes )
for i in range(num_iterations):
#Lets cycle through each input(X) and the corresponding answer (y)
for input,output in zip(X, y):
#Feedforward function gives us the 'guess' for the algorithm
guess = self.feedforward(input)
error = output - guess#How far off were we?
#Now lets take our guess and error and improve our weights for the system
self.backPropagation(input, guess, error)
if i % 100 == 0:
print .5*error**2
#Lets test the input against our weights to see how we did
def predict(self, input=None):
#If we are using our orginal input
if input is None:
inputs = xor['X']
for input in inputs:
input_with_bias = [1] + input
print "Input %s = %s" % ( str(input), self.feedforward( input_with_bias ) )
#If we are testing out another input lets bias it and then run the program
else:
biased_input = addBiasColumn(input)
for bias in biased_input:
print "Input %s = %s" % ( str(bias), self.feedforward( bias ) )
self.weights()
#Lets print out what our weights converged to
def weights(self):
print "Input Weights :\n",self.input_weights
print "Hidden layer weights :\n",self.hidden_layer_weights
def backPropagation(self, input, guess, error):
delta_hidden_nodes = [0]*len(self.hidden_layer_nodes)
#Step 1: Lets get the delta of the output node
delta_output = error * derivative_activate(guess)
#Step 2: Adjust weights for hidden layer -> output
for i in range(len(self.hidden_layer_weights)):#Not vectorizing this so its a little clearer
self.hidden_layer_weights[i] += self.learning_rate*delta_output*self.hidden_layer_nodes[i]
#Step 3: Lets get the hidden layer errors
for i in range(len(self.hidden_layer_nodes)):
out = self.hidden_layer_nodes[i]
delta_hidden_nodes[i] = derivative_activate(out)*( delta_output* self.hidden_layer_weights[i])
#Step 4: Lets change the weights going into the hidden layer
row, column = self.input_weights.shape#(num_hidden_nodes, single_input_size)
for i in range(row):
for j in range(column):
self.input_weights[i][j] += self.learning_rate*delta_hidden_nodes[i]*input[j]
def run():
X, y = xor['X'], xor['y']
nn = neuralNetwork()
nn.fit( X,y)#Lets train the system to get our weights
nn.predict(X)#Lets see how we did
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment