neural_net.py

from __future__ import division
from itertools import izip
from math import exp, sqrt
from random import random
from pprint import pprint

"""
Working through http://neuralnetworksanddeeplearning.com and
implementing it in no-library python.
"""

W = 0 # Weights
B = 1 # Biases
Z = 2 # Weighted inputs
ZD = 3 # Weighted inputs derived
A = 4 # Activations (output)
E = 5 # Errors

def dot(a, b):
    return sum(x * y for x, y in izip(a, b))

def dot_many(xs, b):
    return [dot(x, b) for x in xs]

def minus(a, b):
    return [x - y for x, y in izip(a, b)]

def plus(a, b):
    return [x + y for x, y in izip(a, b)]

def hadamard(a, b):
    return [x * y for x, y in izip(a, b)]

def mult_matrix_vector(m, v):
    return [dot(x, v) for x in m]

def sigmoid(xs):
    return [1/(1+exp(-x)) for x in xs]

def sigmoid_prime(xs):
    return [x * (1-x) for x in xs]

def error_output(network, training):
    network[-1][E] = hadamard(minus(network[-1][A], training), 
                network[-1][ZD])

def error_hidden(network, i):
    network[i][E] = hadamard(
        mult_matrix_vector(network[i+1][W], network[i+1][E]),
        network[i][ZD]
    )

def feed_forward(network, act_func=sigmoid):
    for i, layer in enumerate(network[1:], 1):
        layer[Z] = plus(dot_many(layer[W], network[i-1][A]), layer[B])
        layer[ZD] = sigmoid_prime(layer[Z])
        layer[A] = act_func(layer[Z])

def backpropagate(network, training_data):
    error_output(TEST_NETWORK, training_data)
    for i in xrange(len(TEST_NETWORK)-2, 0, -1):
        error_hidden(TEST_NETWORK, i)

def gradient_descent(network, learning_rate):
    for i in xrange(len(TEST_NETWORK)-1, 0, -1):
        for j, weights in enumerate(network[i][W]):
            for k, weight in enumerate(weights):
                network[i][W][j][k] += (learning_rate * network[i-1][A][k] 
                    * network[i][E][j])
                network[i][B][k] += learning_rate * network[i][E][j]

assert dot([1, 3, -5], [4, -2, -1]) == 3
assert minus([1,4], [1, 5]) == [0, -1]
assert mult_matrix_vector([[1, -1, 2], [0, -3, 1]], [2, 1, 0]) == [1, -3]

TEST_NETWORK = [
    [
        None, # weights
        None, # biases
        None, # weighted inputs
        None, # weighted inputs derirative
        [.05, .10], # outputs
        None, # errors
    ],
    [
        [[.15, .20],[.25, .30]], # weights
        [.35, .35], # biases
        None, # weighted inputs
        None, # weighted inputs derirative
        None, # outputs
        None, # errors
    ],
    [
        [[.40, .45],[.50, .55]], # weights
        [.60, .60], # biases
        None, # weighted inputs
        None, # weighted inputs derirative
        None, # outputs
        None, # errors
    ]
]

i = 0
while True:
    i += 1
    feed_forward(TEST_NETWORK)
    backpropagate(TEST_NETWORK, [.8, .9])
    gradient_descent(TEST_NETWORK, 0.0003)
    if i % 10000 == 0:
        print "Outputs:", TEST_NETWORK[-1][A]
        print "Errors:", TEST_NETWORK[-1][E]
        print