A simple multilayer backprop network.

multilayer.py

#!/usr/bin/python
"""
Author: Jeremy M. Stober
Program: MULTILAYER.PY
Date: Monday, April 14 2008
Description: A simple implementation of a feed-forward multilayer network with back-propagation.
"""

import os, sys, getopt, pdb
from numpy import *
from numpy.random import *
import random as prandom

def logistic(vector):
    return 1 / (1 + exp(-vector))

def dlogistic(vector):
    return vector * (1 - vector)

def sigmoid(vector):
    return tanh(vector)

def dsigmoid(vector):
    return 1 - vector**2

class Multilayer(object):

    def __init__(self, alpha, eta, ninput, nhidden, noutput):

        # Create the weight sets.
        self.input = standard_normal((ninput, nhidden))
        self.output = standard_normal((nhidden, noutput))
        self.storage = {} # Stored activations.
        self.alpha = alpha # Momentum!
        self.eta = eta #Learning rate!
        self.momentum = {'input' : zeros((ninput, nhidden)),
                         'output' : zeros((nhidden, noutput)) }

    def eval(self, input):
        # Use array masks if network is not fully connected.
        # Sum and squash the weights of the input activations.
        hidden = sigmoid(dot(input, self.input))

        # Sum and squash the weights of the hidden activations.
        output = sigmoid(dot(hidden, self.output))

        self.storage = {'input' : input, 'hidden' : hidden, 'output' : output}

        # Output elements are in the range (0,1).
        return output

    def supervise(self, input, response):

        # Will store the activations in self.storage.
        error = response - self.eval(input)

        # Backpropagate the error.
        # Note: * is elementwise multiplication.
        doutput = error * dsigmoid(self.storage['output'])
        dhidden = dot(self.output,doutput) * dsigmoid(self.storage['hidden'])

        # We need to store the last change for momentum.
        deltas = { 'output' : self.eta * outer(self.storage['hidden'],doutput) + self.alpha * self.momentum['output'],
                   'input' :  self.eta * outer(self.storage['input'], dhidden) + self.alpha * self.momentum['input'] }

        self.output = self.output + deltas['output']
        self.input = self.input + deltas['input']
        self.momentum = deltas

        return sum(error ** 2) # Return sum squared error.

def test():

    patterns = [(array([0,1]), array([1])),
                (array([1,0]), array([1])),
                (array([1,1]), array([0])),
                (array([0,0]), array([0]))]

    multi = Multilayer(0.1, 0.5, 2, 2, 1)

    for i in range(10000):
        (input,response) = prandom.choice(patterns)
        multi.supervise(input, response)

    for i in range(10):
        (input,response) = prandom.choice(patterns)
        print input, multi.eval(input), response


def main():

    def usage():
	print sys.argv[0] + "[-h] [-d]"

    try:
        (options, args) = getopt.getopt(sys.argv[1:], 'dh', ['help','debug'])
    except getopt.GetoptError:
        # print help information and exit:
        usage()
        sys.exit(2)

    for o, a in options:
        if o in ('-h', '--help'):
            usage()
            sys.exit()
	elif o in ('-d', '--debug'):
	    pdb.set_trace()

    test()


if __name__ == "__main__":
    main()