mlpmain.cpp

// Simple Multi Layer program ...  for CY2D7
// Has network with 2 inputs, 1 output, tested on XOR problem
// Can have linear or sigmoidal activation
// Also allows linear problem to be solved
// Dr Richard Mitchell 15/12/06 ... 17/3/08 ... 17/4/08
// Adapted by 

#include <iostream>
#include <fstream>
#include "mlplayer.h"
	// define the classes for layers of neurons
#include "filedata.h"
	// define the datasets class


using namespace std;

void passdata (LinActLayer *net, datasets & data, int printall) {
	// pass each item in the data set, data, to network, net, 
	// and store the outputs which were calculated back into data
	// if printall = 0; then just print SSE; 
	// if printall = 1, then also print ins/outs
	// if printall = -1, print nothing
	for (int ct=0; ct<data.numData(); ct++) 
				// for each item in data set
		net -> TestNetwork (data.NthInputs(ct), data.NthOutputs(ct));
				// pass inputs to network and calculate outputs
				// return calculated outputs into data set
				// NB calls to data return address of first 
				// input/output of ct'th item in data set
	if (printall >= 0) data.printdata(printall);
}				// if appropriate print result suitably


void learndata (LinActLayer *net, datasets &data, double learnparas[], int doprint) {
	// pass all of training set in data to network net, and apply training
	// for each item in set, pass to network, calc errors and adjust weights
	// learning rate and momentum are in array learparas
	// if doprint > 0 then print the SSE of the data;
	for (int ct=0; ct<data.numData(); ct++) {	
				// for each item in data set
		net -> TrainNetwork (data.NthInputs(ct), data.NthOutputs(ct), 
							 data.NthTargets(ct), learnparas);
				// pass inputs to network and return calculated outputs
				// the train, based on targets using the learning parameters
				// find addresses of first input/output/target from data set
	}
	if (doprint) {
		cout << "Epoch " << doprint << " ";
		data.printdata(0);
	}
}

void SetTheWeights (LinActLayer *net, int nopt) {
	// if called this initialises network net with specific weights
	// some weights array are defined for xor and three logic probs
	double pictonweights[] = {0.862518, -0.155797,  0.282885,
							  0.834986, -0.505997, -0.864449,
							  0.036498, -0.430437,  0.481210};
								// weights given in Picton's book
	double logweights[] = {0.2, 0.5, 0.3, 0.3, 0.5, 0.1, 0.4, 0.1, 0.2};	
						// initial weights for neuron layer as used in lectures
	if (nopt == 'X')		// if doing XOR problem init with pictonweights
		net -> SetTheWeights (pictonweights);
	else if ( (nopt == 'L') || (nopt == 'S') )	// if doing logic problems, init with others
		net -> SetTheWeights (logweights);
						// otherwise the deafult (random) weights are left unchanged
}

LinActLayer * MakeNet (char nopt, int numhids, datasets &data) {
		// create and return appropriate network type
		// nopt specifies linear activation/sigmoidal or multi layer
		// number of inputs/outputs defined in data set data
		// for multilayer, numhids has number of nodes in hidden layers
	if (nopt == 'L')				// if specify linear layer
		return new LinActLayer (data.numIns(), data.numOuts());
								// call constructor for LinActLayer
	else if (nopt == 'S')			// if specify sigmoidal layer
		return new SigActLayer (data.numIns(), data.numOuts());
								// call constructor for SigActLayer
	else						// if multi-layer
		return new SigActHidLayer (data.numIns(), numhids, 
		                   new SigActLayer (numhids, data.numOuts()) );
								// call constructor for SigActLayer
}								// whose next layer is a SigActLayer

char getcapch(void) {
	// gets next character input by user from console, 
	// if lower case convert to equiv upper case
	char och;
	cin >> och;								// get character
	cin.ignore(1);							// skip rest of line

	if ( (och >= 'a')  && (och <= 'z')) och = och - 32;
											// if lower case convert
	return och;
}

void showweights (LinActLayer *net) {
	// function to print the weights of the neurons in the network net
			// first get space for sufficient number of weights
	double *warray = new double [net->HowManyWeights()];
			// next get the weights from the network
	net->ReturnTheWeights (warray);
			// now print them out in turn
	for (int ct=0; ct<net->HowManyWeights(); ct++) 
		cout << warray[ct] << ',';
	cout << '\n';
			// return array to heap
	delete warray;
}

void testnet (char nopt, int ropt, char *filename, char *dataname, double learnparas[]) {
	// routine to test the network
	// nopt is network option selected (0 = Lin, 1 = Sig, 2 = Multi Layer XOR)
	// ropt is 0 if specific initial weights are to be used, otehrwise default random ones
	// s is name of file with training data
	// lrate and mmtum are the learning rate and momentum for training

	int emax, esofar = 0, ifprint;	// maximum number of epochs, epochs so far and ifprint
	char och = ' ';					// character used for commands input by user

	datasets data (filename, dataname);				// get data sets from file

	if (nopt == 'L') emax = 7; else emax = 1001;
									// if sigmoidal/XOR have 1001 epochs ..

	LinActLayer *net = MakeNet (nopt, 2, data);
									// create appropriate form of network
	if (ropt == 0) SetTheWeights (net, nopt);
			// if not random weights, initialise weights as appropriate.

	passdata (net, data, 1);		// test untrained net and print results

	while (och != 'A') {			// loop as follows depending on user commands
		cout << "Enter L to learn, P to present data, W to find weights, A to abort >";
		och = getcapch();

		if (och == 'L') {			// choose to train ... now learn emax epochs
			for (int ct = 0; ct < emax; ct++) {
				if ( (emax < 10) || (ct % 200 == 0) ) ifprint = ct + esofar; else ifprint = 0;
					// print SSE every 200'th epoch if sigmoidal, else each time
				learndata (net, data, learnparas, ifprint);
					// pass data to network and update weights; print if needed
			}
			esofar = esofar + emax - 1;
		}
		else if (och == 'P')		// choose to pass training data to network and print
			passdata (net, data, 1);	
		else if (och == 'W')		// choose to display weights of neurons
			showweights (net);
	}
}

/**
 * Function that is a bit like passdata but it returns a pointer to a double instead
 * of printing the output, which is far more useful for calculations.
 */
double * get_sse( datasets *data, LinActLayer *net )
{
	for( int ct=0; ct < data->numData(); ct++ )
	{
		net->TestNetwork( data->NthInputs( ct ), data->NthOutputs( ct ) );
	}

	return data->CalcSSE();
}
/**
 * James E. Cleveland's numtest() function.
 * Provides testing for numbers, offers validation set support.
 */
void numtest( double learnparas[], int numhid, int emax, int usevalid, int wopt, char *tstr, char *vstr, char *ustr )
{
	// test network on the numerical problem
	// specified are the learning rate, momentum, number of hidden neurpons
	// emax is max number of epochs for learning
	// if usevalid then stop training when SSE on validation set starts to rise
	// wopt is seed used to initialise random number generators used to initialise weights
	// data files names are in tsr, vstr and ustr
	
	int
		// maximum number of epochs, epochs so far and ifprint
		ifprint		= 0,
		// The number of epochs to take an average over when validating.
		epoch_avg	= 10;
	char
		och		= ' ';
	double
		// A variable to hold SSE 
		*sse,
		// Total SSE for current epoch set
		cur_total	= 0,
		// Total SSE for last epoch set
		last_total	= 99;

	// Seed RNG
	srand( wopt );

	// Load the datasets into seperate objects.
	datasets train	( tstr, "training" );
	datasets unseen	( ustr, "unseen");
	datasets valid	( vstr, "validation" );
	
	// Create a network.
	LinActLayer *net = MakeNet( 'N', numhid, train );

	// Output to describe our current net.
	cout << endl << "************************************"
		<< endl << endl <<
		"Testing a network with " << numhid << " hidden neurons, initial weight of "
		<< wopt << ", learning rate of " << learnparas[ 0 ] << ", and momentum of "
		<< learnparas[ 1 ] << "." << endl << endl;

	// Pass respective data to net to print the SSEs.
	passdata( net, train, 0 );
	passdata( net, unseen, 0 );	

	if( usevalid == 1 )
	{
		passdata( net, valid, 0 );
	}
	
	// Train.
	cout << "Training..." << endl;
	for( int ct = 0; ct < emax; ct++ )
	{
		// Learn the data.
		learndata( net, train, learnparas, ifprint );
		
		// Compare the validation set's SSEs to see if they are rising.
		if( usevalid == 1 )
		{
			// Get current SSE (my function) and add it to the current total.
			sse = get_sse( &valid, net );
			cur_total += *sse;
			if( ct > 150 && ct % epoch_avg == 0 )
			{
				// Compare the totals...no real point averaging as they're both divided by epoch_avg anyway.
				if( cur_total > last_total )
				{
					// Set the CT to emax so we don't loop again.
					ct = emax;
				}
				// Set the last_total to the current total and reset the current total to zero.
				last_total = cur_total;
				cur_total = 0;
			}
		}
		
	}

	// Pass respective data to net to see the SSE.
	passdata( net, train, 0 );
	passdata( net, unseen, 0 );

	if( usevalid == 1 )
	{
		passdata( net, valid, 0 );
	}
}

void main()
{
					// function to create and test single/multi layer network
	int wopt = 0, hopt = 10, emax = 1001;		// initialse key options
	double learnparas[2]; learnparas[0] = 0.2; learnparas[1] = 0.5;
	char och = 'A', nopt = 'L', usevalid = 'Y';

	cout << "RJMs Perceptron network adapted by RJM and extended by James E. Cleveland\n";

	while (och != 'Q') {						// loop til quit
		cout << "\nNetwork is ";				// display current set up
		if (nopt == 'L') cout << "Linear Layer; ";
		else if (nopt == 'S') cout << "Sigmoidal Layer; ";
		else if (nopt == 'X') cout << "for XOR; ";
		else cout << "for Lin Prob with " << hopt << " hidden neurons; ";
		cout << "\nInit weights seed " << wopt << " ";
		cout << "Learning rate " << learnparas[0] << " momentum " << learnparas[1] << "\n";

		cout << "Choose (T)est network, set (N)etwork, set learning (C)onstants, (I)nitialise weights, or (Q)uit  (V)=numtest1 > ";
												// specify user's options
		och = getcapch();	
		if( och == 'V' )
		{
			learnparas[0] = 0.2; learnparas[1] = 0.5;
			numtest (learnparas, 10, 200, 1, wopt, "train.txt", "valid.txt", "unseen.txt");
			learnparas[ 0 ] = 0.1; learnparas[ 1 ] = 0.6;
			numtest (learnparas, 10, 200, 1, 0, "train.txt", "valid.txt", "unseen.txt");
			numtest (learnparas, 10, 200, 1, 10, "train.txt", "valid.txt", "unseen.txt");
			numtest (learnparas, 10, 200, 1, 25, "train.txt", "valid.txt", "unseen.txt");
			numtest (learnparas, 10, 200, 1, 37, "train.txt", "valid.txt", "unseen.txt");
			numtest (learnparas, 15, 200, 1, 0, "train.txt", "valid.txt", "unseen.txt");
			numtest (learnparas, 15, 200, 1, 10, "train.txt", "valid.txt", "unseen.txt");
			learnparas[ 0 ] = 0.1; learnparas[ 1 ] = 0.4;
			numtest (learnparas, 15, 200, 1, 0, "train.txt", "valid.txt", "unseen.txt");
			learnparas[ 0 ] = 0.15; learnparas[ 1 ] = 0.4;
			numtest (learnparas, 15, 200, 1, 0, "train.txt", "valid.txt", "unseen.txt");
		}					// read user's choice
		if( och == 'B' )
		{
			learnparas[0] = 0.1; learnparas[1] = 0.6;
			numtest (learnparas, 10, 200, 0, wopt, "train.txt", "valid.txt", "unseen.txt");
		}					// read user's choice
		if( och == 'M' )
		{
			learnparas[0] = 1; learnparas[1] = 1;
			numtest (learnparas, 100, 500, 0, wopt, "train.txt", "valid.txt", "unseen.txt");
		}					// read user's choice
		if (och == 'N') {						// user to choose network
			cout << "For Network enter (L)inear layer (S)igmoidal (X)OR else (N)umerical Prob > ";
			nopt = getcapch();// get network type
			if ( (nopt != 'L') && (nopt != 'S') && (nopt != 'X')) {
			    cout << "Enter number of nodes in hidden layer > ";
				cin >> hopt; // if approp, get num hidden nodes also
				cin.ignore(1);
				cout << "Enter max number of epochs for learning >";
				cin >> emax; // if approp, get num hidden nodes also
				cin.ignore(1);
				cout << "Use validation set to stop learning (Y/N) > ";
				usevalid = getcapch();
			}
		}
		else if (och == 'C') {					// user to set learning rate and momentum
			cout << "Enter l-rate & momentum (both in range 0..1) > ";
			cin >> learnparas[0] >> learnparas[1];
			cin.ignore(1);
		}
		else if (och == 'I') {					// user to specify initial weights
			if ( (nopt != 'L') && (nopt != 'S') && (nopt != 'X')) 
				cout << "Enter 0 to use weights in notes, else set random weights> ";
			else
				cout << "Enter seed used for random weights> ";
   			cin >> wopt;
			cin.ignore(1);
		}
		else if (och == 'T') {					// option to test network
			if ( (nopt == 'L') || (nopt == 'S') )	// test single layer network
				testnet (nopt, wopt, "logdata.txt", "AndOrXor", learnparas);
			else if (nopt == 'X')				// test MLP on XOR
				testnet (nopt, wopt, "xordata.txt", "XOR", learnparas);
			else								// test on numerical problem
				numtest (learnparas, hopt, emax, usevalid == 'Y', wopt, "train.txt", "valid.txt", "unseen.txt");
			printf( "TEST FINISH\n" );
		}
	}
}