CI 4_3

ci_4_3.m

%%%
% Feed forward with back propagation algorithm
%
% @param    X ... all possible inputs (matrix)
% @param    Y ... all expected outputs (matrix)
% @param    T ... the network topology (matrix)
% @param    l ... the iteration limit for the error min. func.
% @returns  W ... the calculated weight matrix
% @returns Ev ... the error dynamic vector
%%
function [W, Ev] = ci_4_3(X, Y, T, l)
	
	dim = length(T);
	y = zeros(1, dim);
	W = randn(dim,dim);
	nn = 0.2;
 	Ev = out = [];
	epochs = 0;
	
	while(1)		
		E = 0;
		for i=1:length(Y)
			n = dim;
			y_i = forwardProp(W, T, X(i,:));
			out(i) = y_i(n);
			e = ErrorFunction(y_i(n), Y(i));
			E = E + e;	
			
			w = [];
			k(dim) = deltaOutput(y_i(dim), Y(i));
			w(:,dim) = k(dim)*y_i;
			for i=dim-size(Y, 2):-1:1
				k(i) = deltaHidden(T, W, k, y_i(i), i);
				w(:,i) = k(i)*y_i;		
			end;
			
			W = W - nn*w;
		end;
		
		Ev = [Ev, E];
		
		% print iteration with current error
		printf('%d:\t%f\n', epochs, E);
		fflush(stdout);
		
		% stop if iteration limit is reached
		if (++epochs > l)
			break;
		end;
	end;
	
	% "remove" unused weights for a more readable output
	W = W.*T;
	
	% plot the error dynamic
	semilogy(Ev);
	

	
	function e = ErrorFunction(y_i, y)
		e = (y_i - y)^2;	
	end;
	
	function f = F_(y)
		f = y*(1-y);
	end;
	
	function k = deltaOutput(y, d)
		k = 2*(y-d)*F_(y);
	end;
	
	function k = deltaHidden(T, W, k, y, n)
		dim = length(T);
		sum = 0;
		for j = 1:dim
			if (T(n,j) == 1)
				sum = sum + W(n, j)*k(j);
			end;
		end;
		k = F_(y)*sum;
	end;
	
end;

ci_4_3_calling.m

l = 1000; %iteration limits

%---------Solving XOR------------
X = [ 0 0 ;
      0 1 ;
      1 0 ;
      1 1];
Y = [0; 1; 1; 0];

%---------1 Hidden Layer, 4 Hidden Neurons 	

% (1)--|-----(3)----|
%      |-----(4)----|
%      |	    |---(7)
%      |-----(5)----|
% (2)--|-----(6)----|

T =[0 0 1 1 1 1 0;
    0 0 1 1 1 1 0;
    0 0 0 0 0 0 1;
    0 0 0 0 0 0 1;
    0 0 0 0 0 0 1;
    0 0 0 0 0 0 1;
    0 0 0 0 0 0 0 ];

ci_4_3(X,Y,T,l);
	
%---------1 Hidden Layer, 3 Hidden Neurons 	

% (1)--|-----(3)----|
%      |-----(4)----|---(6)
% (2)--|-----(5)----|

T =[0 0 1 1 1 0;
    0 0 1 1 1 0;
    0 0 0 0 0 1;
    0 0 0 0 0 1;
    0 0 0 0 0 1;
    0 0 0 0 0 0 ];

ci_4_3(X,Y,T,l);

%---------1 Hidden Layer, 2 Hidden Neurons 	

% (1)--|-----(3)----|
%      |            |---(5)
% (2)--|-----(4)----|

T =[0 0 1 1 0;
    0 0 1 1 0;
    0 0 0 0 1;
    0 0 0 0 1;
    0 0 0 0 0 ];

ci_4_3(X,Y,T,l);

%---------Minimal Version NN51

% (1)--|-----------|
%      |	   |
% (2)--|-----(4)---|---(5)
%      |	   |
% (3)--|-----------|

X = [ 0 0 1;
      0 1 1;
      1 0 1;
      1 1 1];
	  
T =[0 0 0 1 1;
    0 0 0 1 1;
    0 0 0 1 1;
    0 0 0 0 1;
    0 0 0 0 0 ];

ci_4_3(X,Y,T,l);

%---------Solving Classification------------

%---------2 Hidden Layer, 5 Hidden Neurons per Layer

load K40M2-NN.dat

X = K40M2_NN;
Y = [ones(1,20), zeros(1,20)]';

% (1)--|----(3)----|----( 8)----|
%      |----(4)----|----( 9)----|
%      |----(5)----|----(10)----|--(13)   
%      |----(6)----|----(11)----|   
% (2)--|----(7)----|----(12)----|

T = [0 0 1 1 1 1 1 0 0 0 0 0 0; 
     0 0 1 1 1 1 1 0 0 0 0 0 0; 
     0 0 0 0 0 0 0 1 1 1 1 1 0; 
     0 0 0 0 0 0 0 1 1 1 1 1 0; 
     0 0 0 0 0 0 0 1 1 1 1 1 0; 
     0 0 0 0 0 0 0 1 1 1 1 1 0; 
     0 0 0 0 0 0 0 1 1 1 1 1 0; 
     0 0 0 0 0 0 0 0 0 0 0 0 1; 
     0 0 0 0 0 0 0 0 0 0 0 0 1; 
     0 0 0 0 0 0 0 0 0 0 0 0 1; 
     0 0 0 0 0 0 0 0 0 0 0 0 1; 
     0 0 0 0 0 0 0 0 0 0 0 0 1; 
     0 0 0 0 0 0 0 0 0 0 0 0 0];

[W, Ev] = ci_4_3(X,Y,T,l);

x = -10:4;
y = -10:10;
n = length(T);
o = 11; %offset
x = x + o;
y = y + o;

for i = x
	for j = y
        y_i = forwardProp(W, T, [i-o,j-o]);
        z(j,i) = y_i(n);
	end;
end;

hold on;
    figure(2);
    plot([X(1:20,1)+o],[X(1:20,2)+o], 'ro');
    plot([X(21:40,1)+o],[X(21:40,2)+o], 'bo');
    contour(x, y, z, 10);
hold off;

forwardProp.m

function y = forwardProp(W,T,X)
    dim = length(T);
    y = [X, zeros(1,dim-length(X))];
    for i=length(X)+1:dim
        y(i) = logActivationFunc(propFunc(W(:,i), y, T(:,i)), 1);
    end;
end;

logActivationFunc.m

function x = logActivationFunc(s, lambda)
    x = 1/(1+exp(-lambda*s));
end;

propFunc.m

function p = propFunc(w, y, t)
    p = w' * (t'.*y)';
end;