gistfile1.matlab

% Classification using multilayer perceptron
% Use the data provided (modeldata.txt). The first 
%    two columns are x1 and x2, the column 3 to 5 represent 
%    coding for three class (y).
% Write your own MLP. Use off-line learning, learning 
%    rate=0.001, learning time=1000 step, # of hidden neuron=5,
%    use tanh as your activation function. Plot MSE vs learning 
%    steps. Calculate the min(MSE) and associated optimal weights.
% (NOTE: you should try different initial conditions several times 
% to check whether you get trapped in the local minimum.)

clear;clc;
%% load the modeldata.txt first
modeldata=load("modeldata.txt");

% separate data into training set and testing set. 
train_set=modeldata(1:300,:);
test_set=modeldata(301:end,:);

%% Create training set and testing set
% 1.1 create training set in matrix form
% input:  D: x1,x2,x0 (fill in unity): 300x3
% output: Y: y1,y2,y3 : 300x3
D=[train_set(:,1:2), ones(300,1)];
Y=train_set(:,3:5);

% 1.2 create testing set
% input:  d (same dimension as training set)
% output: y (same dimension as testing set)
d=[test_set(:,1:2), ones(300,1)];
y=test_set(:,3:5);

%% 2. Initialize weights: v (3x5) and w (6x3)

%% Several trials of initial values
%v=[-3,-3,-4,-1,1;2,-3,-1,-2,-3;-1,3,-3,0.5,1];
%w=[2,-1,6;-3,1,2;7,5,4;2,-3,-1;-4,5,2;-2,1,6];
%v=1-2*rand(3,5);
%w=[2,-1,6;-3,1,2;7,5,4;2,-3,-1;-4,5,2;-2,1,6];
%v=1-2*rand(3,5);
%w=1-2*rand(6,3);
%v=[-3,-2,-3,-4,-5;-1,-2,-1,-1,1;3,2,-3,0,1];
%w=[-1,0,-1;-1,-2,-1;-1,1,-1;-2,-2,2;1,-1,1;2,1,2];
%v=[0.1,-0.1,0.7,-0.3,-0.2;0.5,-0.5,-0.4,0.2,0.9;-0.1,-0.3,-0.7,0.5,0.4];
%w=[0.5,-0.3,0.1;-0.7,0.2,-0.3;-0.5,0.4,-0.1;0.6,0.1,0.5;0.8,-0.9,-0.4;0.5,0.2,0.1];
nhid=5;
n_input=2;
n_output=3;
v=-1+2*rand(n_input+1,nhid);
w=-1+2*rand(nhid+1,n_output);

% 2.1 create container of mse
mse_1=zeros(4,1);
mse_2=zeros(4,2);

%% 3. activation function:tanh
activ=inline('tanh(x)','x');
d_act=inline('sech(x).^2','x');

% learning rate (alpha) = 0.001;
% learning time (t) = 1000;
alpha=0.001;
t=1000;

nr_size(Y)(:,1)

for i=1:t
    %% training
    % calculate the first layer: input(D)*weight(v)
    % 300x3 * 3x5 ==> 300 x 5
    uj=D*v;
    % initial activating, forward into hidden layer
    zj0=activ(uj);
    % add bias term 1 zj: 300x6
    zj=[zj0, ones(300,1)];
    % calculate second layer, output
    %   aj=hidden layer results*weight(w): 300x3
    aj=zj*w;
    % second layer activation
    Yh=activ(aj);
    % calculate mean square error
    mse_tr0=[i,1/300*sumsq(Y-Yh)];
    mse_1=[mse_1,mse_tr0'];
    
    %% testing
    ui=d*v;
    zi0=activ(ui);
    zi=[zi0, ones(300,1)];
    ai=zi*w;
    yh=activ(ai);
    mse_test0=[i,1/300*sumsq(y-yh)];
    mse_2=[mse_2,mse_test0'];

    %% update weight
    DELTA=d_act(aj).*(Y-Yh);
    wp=w+alpha*zj'*DELTA;
    delta=d_act(uj).*(DELTA*w(1:5,:)');
    vp=v+alpha*(D'*delta);

    vc(:,:,i)=v;
    wc(:,:,i)=w;

    v=vp;
    w=wp;

end
figure(1)
plot(mse_1(2,:), '.')
xlabel("No. of iteration")
ylabel("MSE")
print('../figures/mse1_1.pdf','-dpdf')
figure(2)
plot(mse_1(3,:), '.')
xlabel("No. of iteration")
ylabel("MSE")
print('../figures/mse1_2.pdf','-dpdf')
figure(3)
plot(mse_1(4,:), '.')
xlabel("No. of iteration")
ylabel("MSE")
print('../figures/mse1_3.pdf','-dpdf')


figure(4)
plot(mse_2(2,:), '.')
xlabel("No. of iteration")
ylabel("MSE")
print('../figures/mse2_1.pdf','-dpdf')
figure(5)
plot(mse_2(3,:), '.')
xlabel("No. of iteration")
ylabel("MSE")
print('../figures/mse2_2.pdf','-dpdf')
figure(6)
plot(mse_2(4,:), '.')
xlabel("No. of iteration")
ylabel("MSE")
print('../figures/mse2_3.pdf','-dpdf')

% optw=[wc(:,1:2,1000),wc(:,3,670)];
% optv=[vc(1:2,:,1000);vc(3,:,670)];