network.py

from numpy import exp, random, zeros, reshape
from time import time

# Sigmoid (Logistic) Activation Function
def sigmoid (x):
        return 1 / (1 + exp(-x))

# Derivative of the Sigmoid Function
def sigmoid_derivative (z):
        return z * (1 - z)

# Derivative of the Mean Squared Error Cost Function
def mse_derivative (output, target):
        return output - target

# Neural Network
class Network:
        # Initialise the network
        def __init__ (self, sizes):
                # Good Practice
                random.seed(1)
        
                # Number of features; number of neurons in the input layer
                self.no_features = sizes[0]

                # The number of weight matrices
                # Basically the number_of_network_layers - 1
                self.btw_layers = len(sizes) - 1
        
                # Randomly initialize the weights
                # Initialise the biases to zeros
                self.biases = [ zeros((a, 1)) for a in sizes[1:] ]
                self.weights = [ 2 * random.randn(a, b) - 1 for a, b in zip(sizes[:-1], sizes[1:]) ]

                # Deltas (error_derivatives), with respect to the parameters
                # Initialized to zeros
                self.weight_deltas = [ zeros(w.shape) for w in self.weights ]
                self.bias_deltas = [ zeros(b.shape) for b in self.biases ]

        # Evaluate the accuracy on some data
        def evaluate (self, test_data):
                accuracy = 0.0
                for x_test, y_test in test_data:
                        act = reshape(x_test, (self.no_features, 1))
                        for l in xrange(self.btw_layers):
                                act = sigmoid(self.weights[l].T.dot(act) + self.biases[l])
                        if round(act) == y_test:
                                accuracy += 1
                return accuracy

        # Train the network using some parameters
        def train (self, training_data, epochs=3000, learning_rate=5.0, regularisation_parameter=0.0, mini_batch_size=None, check_at=10, test_data=None):
                total_trainig_data = len(training_data)
                
                if mini_batch_size == None: mini_batch_size = total_trainig_data

                # shuffle the training_data
                random.shuffle(training_data)

                # Split the data into mini-batches
                mini_batches = [ training_data[k:k+mini_batch_size] for k in xrange(0, total_trainig_data, mini_batch_size) ]

                start_time = time()
                for iter in xrange(epochs):
                        for mini_batch in mini_batches:
                                batch_total = len(mini_batch)
                                for x, y in mini_batch:
                                        activation = reshape(x, (self.no_features, 1))
                                        activations = [activation]

                                        # Forward Propagation (Feed-Forward)
                                        for we, bi in zip(self.weights, self.biases):
                                                activation = sigmoid(we.T.dot(activation) + bi)
                                                activations.append(activation)

                                        # Error Derivative
                                        error_derivative_wrt_output_activation = mse_derivative(activations[-1], y)
                                        delta = error_derivative_wrt_output_activation * sigmoid_derivative(activations[-1])

                                        # Back-Propagation
                                        for l in xrange(self.btw_layers):
                                                self.bias_deltas[-l-1] = delta
                                                self.weight_deltas[-l-1] = activations[-l-2].dot(delta.T)
                                                delta = (self.weights[-l-1] * sigmoid_derivative(activations[-l-2])).dot(delta)

                                        # Update Parameters (Weights and Biases)
                                        for l in xrange(self.btw_layers):
                                                self.biases[l] = self.biases[l] - ((learning_rate / batch_total) * self.bias_deltas[l])
                                                self.weights[l] = ((1 - (learning_rate * (regularisation_parameter/batch_total))) * self.weights[l]) - ((learning_rate / batch_total) * self.weight_deltas[l])
                        
                        if iter % check_at == 0:
                                if test_data != None:
                                        test_total = len(test_data)
                                        accuracy = self.evaluate(test_data)
                                        print "Accuracy: {0}/{1} => {2}%".format(accuracy, test_total, 100*(accuracy/test_total))
                                        print "After {0} epoch(s), in {1} seconds\n".format(iter, time() - start_time)