An implementation example of multilayer perceptron (MLP) in pure ES6 JavaScript without third-party libraries

MLP-simple.js

/**
 * A program to create, train, and use a simple MLP
 * 
 * Below is an implementation example of multilayer perceptron (MLP) in pure ES6
 * JavaScript without third party libraries.
 * 
 * @author srifqi
 * @license MIT License
 */

/** ======= MAIN COMPONENTS' FUNCTIONS ======= **/

/**
 * @typedef MLP
 * @type {object}
 * @property {number[][]} layers Interlayer weight matrix
 * @property {number[][]} bias Bias weight matrix for each layer (after input layer)
 * @property {function[]} activations List of activation functions of each layer (after input layer)
 * @property {function[]} derActivations List of activation functions' (first) derivation of each layer (after input layer)
 */

/**
 * Main function to create an MLP
 * 
 * @param {number[]} sizes List of sizes for each layer (including input layer)
 * @param {number[]} activations List of activation functions for each layer (after input layer)
 * @returns {MLP} An MLP model with the given parameters
 */
function createMLP(sizes, activations) {
	const model = {
		layers: [],
		bias: [],
		activations: [],
		derActivations: []
	};
	for (let i = 1; i < sizes.length; i ++) {
		const sA = sizes[i - 1];
		const sB = sizes[i];
		model.layers.push(createRandom2DMatrix(sB, sA));
		model.bias.push(createRandom2DMatrix(sB, 1));
	}
	for (let i = 0; i < activations.length; i ++) {
		const ai = activations[i];
		model.activations.push(listGenActivations[ai]);
		model.derActivations.push(listDerActivations[ai]);
	}
	return model;
}

/**
 * Main function to train an MLP
 * 
 * The training is done stochastically, i.e. the parameter update is done for
 * each entry and not using its average error.
 * 
 * @param {MLP} model Model to be trained
 * @param {number[][]} X List of inputs from the training dataset
 * @param {number[][]} Y List of outputs (results/targets) from the training dataset
 * @param {Function} learningRateFunction Function to calculate learning rate constant
 * @param {number} maxIter Amount of maximum iteration
 * @returns {MLP} Trained model
 */
function train(model, X, Y, learningRateFunction, maxIter) {
	if (X.length != Y.length) {
		throw new Error(`The size of X and Y doesn't match: ${X.length} with ${Y.length}.`);
	}
	let lastLearningRate = 0;
	// each iteration
	for (let iter = 1; iter <= maxIter; iter ++) {
		const learningRate = learningRateFunction(iter - 1);
		if (lastLearningRate != learningRate) {
			console.log(`Learning rate is set to ${learningRate}.`);
			lastLearningRate = learningRate;
		}
		let correct = 0;
		let totalError = 0;
		// each entry
		for (let k = 0; k < X.length; k ++) {
			// feed-forward
			const inputs = transpose([X[k]]);
			const listOfValues = feedForward(model, inputs);
			const outputs = listOfValues[listOfValues.length - 1];
			// to calculate accuracy
			if (argMax(transpose(outputs)[0]) == argMax(Y[k])) {
				correct ++;
			}
			// back-propagation
			let error = substract(outputs, transpose([Y[k]]));
			totalError += transpose(error)[0].reduce((a, v) => a + v);
			// no multiplication with output layer's activation function's derivation
			for (let i = model.layers.length - 1; i >= 0; i --) {
				const layerError = multiply(
					error,
					transpose(listOfValues[i]),
					learningRate
				);
				const layerBiasError = multiply(
					error,
					learningRate
				)
				model.layers[i] = substract(model.layers[i], layerError);
				model.bias[i] = substract(model.bias[i], layerBiasError);
				if (i > 0) {
					error = multiply(transpose(model.layers[i]), model.derActivations[i - 1](error));
				}
			}
		}
		const accuracy = correct / X.length;
		const avgError = totalError / X.length;
		console.log(`Iteration ${iter}/${maxIter} | Error: ${avgError} | Accuracy: ${correct}/${X.length} / ${accuracy}`);
	}
	return model;
}

/**
 * Main function to predict using given MLP
 * 
 * @param {MLP} model Model that will predict
 * @param {number[][]} X List of inputs
 * @returns {Object.<string,*>}
 */
function predict(model, X) {
	const result = {
		outputs: [],
		selected: []
	}
	for (let k = 0; k < X.length; k ++) {
		const inputs = transpose([X[k]]);
		const listOfValues = feedForward(model, inputs);
		const outputs = listOfValues[listOfValues.length - 1];
		const selected = argMax(transpose(outputs)[0]);
		result.outputs.push(outputs);
		result.selected.push(selected);
	}
	return result;
}

/** ======= MATHEMATICS HELPER FUNCTIONS ======= **/

/**
 * Helper function to create a 2 dimensional matrix with a uniform fill
 * 
 * @param {number} rows Amount of rows
 * @param {number} columns Amount columns
 * @param {number} [values=0] Value for each matrix's element
 * @returns {number} New matrix using given parameters
 */
function create2DMatrix(rows, columns, values) {
	if (values == undefined) {
		values = 0;
	}
	return Array.from(Array(rows), () => Array.from(Array(columns), () => values));
}

/**
 * Helper function to create a 2 dimensional matrix with a random fill
 * 
 * @param {number} rows Amount of rows
 * @param {number} columns Amount columns
 * @returns {number} New matrix using given parameters
 */
function createRandom2DMatrix(rows, columns) {
	return Array.from(Array(rows), () => Array.from(Array(columns), () => Math.random()));
}

/**
 * Helper function to transpose a matrix
 * 
 * @param {number[][]} matrix Matrix to be transposed
 * @returns Transposed matrix
 */
function transpose(matrix) {
	let result = create2DMatrix(matrix[0].length, matrix.length);
	for (let i = 0; i < matrix.length; i ++) {
		for (let j = 0; j < matrix[0].length; j ++) {
			result[j][i] = matrix[i][j];
		}
	}
	return result;
}

/**
 * Helper function to multiply many values
 * 
 * Given values can be a matrix or a scalar.
 * 
 * @param  {...number|number[][]} elements List of values to be multiplied
 * @returns {number|number[][]} Result of multiplication
 */
function multiply(...elements) {
	if (elements.length == undefined || elements.length <= 1) {
		throw new Error("There is only one value that will be multiplied.");
	}
	let result = elements[0];
	for (let q = 1; q < elements.length; q ++) {
		let ki = result;
		let ka = elements[q];
		// both are scalars
		if (ki.length == undefined && ka.length == undefined) {
			result = ki * ka;
		// both are matrices
		} else if (ki.length >= 1 && ka.length >= 1) {
			if (ki[0].length != ka.length) {
				let kiy = ki.length;
				let kix = ki[0].length;
				let kay = ka.length;
				let kax = ka[0].length;
				throw new Error(`Matrices' size doesn't match: ${kiy}x${kix} with ${kay}x${kax}.`);
			}
			result = create2DMatrix(ki.length, ka[0].length);
			for (let i = 0; i < ki.length; i ++) {
				for (let j = 0; j < ka[0].length; j ++) {
					let total = 0;
					for (let k = 0; k < ka.length; k ++) {
						total += ki[i][k] * ka[k][j];
					}
					result[i][j] = total;
				}
			}
		// one of those is a matrix
		} else {
			let scalar = ki;
			let matrix = ka;
			if (ka.length == undefined) { // only check one of those
				scalar = ka;
				matrix = ki;
			}
			for (let i = 0; i < matrix.length; i ++) {
				for (let j = 0; j < matrix[0].length; j ++) {
					matrix[i][j] *= scalar;
				}
			}
			result = matrix;
		}
	}
	return result;
}

/**
 * Helper function to add two matrices
 * 
 * Both matrices need to have the same size.
 * 
 * @param {number[][]} A Matrix A
 * @param {number[][]} B Matrix B
 * @returns {number[][]} Result of addition between two matrices: A + B.
 */
function addition(A, B) {
	if (A.length != B.length || A[0].length != B[0].length) {
		let Ay = A.length;
		let Ax = A[0].length;
		let By = B.length;
		let Bx = B[0].length;
		throw new Error(`Matrices' size are not the same: ${Ay}x${Ax} with ${By}x${Bx}.`);
	}
	for (let i = 0; i < A.length; i ++) {
		for (let j = 0; j < A[0].length; j ++) {
			A[i][j] += B[i][j];
		}
	}
	return A;
}

/**
 * Helper function to substract two matrices
 * 
 * Both matrices need to have the same size.
 * 
 * @param {number[][]} A Matrix A
 * @param {number[][]} B Matrix B
 * @returns {number[][]} Result of substraction from matrix A by matrix B: A - B.
 */
function substract(A, B) {
	if (A.length != B.length || A[0].length != B[0].length) {
		let Ay = A.length;
		let Ax = A[0].length;
		let By = B.length;
		let Bx = B[0].length;
		throw new Error(`Matrices' size are not the same: ${Ay}x${Ax} with ${By}x${Bx}.`);
	}
	for (let i = 0; i < A.length; i ++) {
		for (let j = 0; j < A[0].length; j ++) {
			A[i][j] -= B[i][j];
		}
	}
	return A;
}

/**
 * Helper function to find an index which has the maximum value
 * 
 * If there are many maximum values, the lowest index is chosen.
 * 
 * @param {number[]} list List that will be searched
 * @returns {number} Index which has the maximum value
 */
function argMax(list) {
	if (list.length == undefined) {
		throw new Error("Input is not a list.");
	}
	if (list.length == 0) {
		throw new Error("Given list is empty.");
	}
	let maxIndex = 0;
	for (let q = 1; q < list.length; q ++) {
		if (list[q] > list[maxIndex]) {
			maxIndex = q;
		}
	}
	return maxIndex;
}

/**
 * Helper function to do feed-forward
 * 
 * @param {MLP} model Model which will be used
 * @param {number[][]} X List of inputs
 * @returns {number[][]} List of outputs for each layer (after input layer)
 */
function feedForward(model, X) {
	let values = X;
	let listOfResults = [values];
	for (let i = 0; i < model.layers.length; i ++) {
		values = multiply(model.layers[i], values);
		values = addition(values, model.bias[i]);
		values = model.activations[i](values);
		listOfResults.push(values);
	}
	return listOfResults;
}

/**
 * List of activation functions
 * 
 * @type {Object<string,Function>}
 */
let listGenActivations = {};

/**
 * List of activation functions' derivation
 * 
 * @type {Object<string,Function>}
 */
let listDerActivations = {};

/**
 * Helper function to apply ReLU for each element in a matrix
 * 
 * @param {number[][]} x Input matrix
 * @returns {number[][]} Resulting matrix after ReLU mapping
 */
function genReLU(x) {
	for (let i = 0; i < x.length; i ++) {
		for (let j = 0; j < x[i].length; j ++) {
			x[i][j] = Math.max(x[i][j], 0);
		}
	}
	return x;
}
listGenActivations["relu"] = genReLU;

/**
 * Helper function to apply ReLU's derivation for each element in a matrix
 * 
 * @param {number[][]} x Input matrix
 * @returns {number[][]} Resulting matrix after ReLU's derivation mapping
 */
function derReLU(x) {
	for (let i = 0; i < x.length; i ++) {
		for (let j = 0; j < x[i].length; j ++) {
			x[i][j] = x[i][j] >= 0 ? 1 : 0;
		}
	}
	return x;
}
listDerActivations["relu"] = derReLU;

/** ======= MAIN PROGRAM ======= **/

/**
 * Main function that will be run first (see last section)
 */
function main() {
	const sizes = [4, 5, 3];
	const activations = ["relu", "relu"]; // only ReLU
	const model = createMLP(sizes, activations);
	const constantLearningRate = () => 0.001;
	train(model, X, Y, constantLearningRate, 4);
	console.log(predict(model, [X[0]])); // prediction example
}

/** ======= PROGRAM EXECUTION ======= **/

// Example of Iris dataset (see below)
const X = [ // list of inputs (n, w, h)
	[5.1, 3.5, 1.4, 0.2]
	// ...
]; // list of inputs (n, w, h)
const Y = [ // list of label, one-hot encoding (n, j)
	[1, 0, 0]
	// ...
]; // list of label, one-hot encoding (n, j)

main();

set-iris.js

const X = [ // list of inputs (n, w, h)
	[5.1,3.5,1.4,0.2],
	[4.9,3.0,1.4,0.2],
	[4.7,3.2,1.3,0.2],
	[4.6,3.1,1.5,0.2],
	[5.0,3.6,1.4,0.2],
	[5.4,3.9,1.7,0.4],
	[4.6,3.4,1.4,0.3],
	[5.0,3.4,1.5,0.2],
	[4.4,2.9,1.4,0.2],
	[4.9,3.1,1.5,0.1],
	[5.4,3.7,1.5,0.2],
	[4.8,3.4,1.6,0.2],
	[4.8,3.0,1.4,0.1],
	[4.3,3.0,1.1,0.1],
	[5.8,4.0,1.2,0.2],
	[5.7,4.4,1.5,0.4],
	[5.4,3.9,1.3,0.4],
	[5.1,3.5,1.4,0.3],
	[5.7,3.8,1.7,0.3],
	[5.1,3.8,1.5,0.3],
	[5.4,3.4,1.7,0.2],
	[5.1,3.7,1.5,0.4],
	[4.6,3.6,1.0,0.2],
	[5.1,3.3,1.7,0.5],
	[4.8,3.4,1.9,0.2],
	[5.0,3.0,1.6,0.2],
	[5.0,3.4,1.6,0.4],
	[5.2,3.5,1.5,0.2],
	[5.2,3.4,1.4,0.2],
	[4.7,3.2,1.6,0.2],
	[4.8,3.1,1.6,0.2],
	[5.4,3.4,1.5,0.4],
	[5.2,4.1,1.5,0.1],
	[5.5,4.2,1.4,0.2],
	[4.9,3.1,1.5,0.1],
	[5.0,3.2,1.2,0.2],
	[5.5,3.5,1.3,0.2],
	[4.9,3.1,1.5,0.1],
	[4.4,3.0,1.3,0.2],
	[5.1,3.4,1.5,0.2],
	[5.0,3.5,1.3,0.3],
	[4.5,2.3,1.3,0.3],
	[4.4,3.2,1.3,0.2],
	[5.0,3.5,1.6,0.6],
	[5.1,3.8,1.9,0.4],
	[4.8,3.0,1.4,0.3],
	[5.1,3.8,1.6,0.2],
	[4.6,3.2,1.4,0.2],
	[5.3,3.7,1.5,0.2],
	[5.0,3.3,1.4,0.2],
	[7.0,3.2,4.7,1.4],
	[6.4,3.2,4.5,1.5],
	[6.9,3.1,4.9,1.5],
	[5.5,2.3,4.0,1.3],
	[6.5,2.8,4.6,1.5],
	[5.7,2.8,4.5,1.3],
	[6.3,3.3,4.7,1.6],
	[4.9,2.4,3.3,1.0],
	[6.6,2.9,4.6,1.3],
	[5.2,2.7,3.9,1.4],
	[5.0,2.0,3.5,1.0],
	[5.9,3.0,4.2,1.5],
	[6.0,2.2,4.0,1.0],
	[6.1,2.9,4.7,1.4],
	[5.6,2.9,3.6,1.3],
	[6.7,3.1,4.4,1.4],
	[5.6,3.0,4.5,1.5],
	[5.8,2.7,4.1,1.0],
	[6.2,2.2,4.5,1.5],
	[5.6,2.5,3.9,1.1],
	[5.9,3.2,4.8,1.8],
	[6.1,2.8,4.0,1.3],
	[6.3,2.5,4.9,1.5],
	[6.1,2.8,4.7,1.2],
	[6.4,2.9,4.3,1.3],
	[6.6,3.0,4.4,1.4],
	[6.8,2.8,4.8,1.4],
	[6.7,3.0,5.0,1.7],
	[6.0,2.9,4.5,1.5],
	[5.7,2.6,3.5,1.0],
	[5.5,2.4,3.8,1.1],
	[5.5,2.4,3.7,1.0],
	[5.8,2.7,3.9,1.2],
	[6.0,2.7,5.1,1.6],
	[5.4,3.0,4.5,1.5],
	[6.0,3.4,4.5,1.6],
	[6.7,3.1,4.7,1.5],
	[6.3,2.3,4.4,1.3],
	[5.6,3.0,4.1,1.3],
	[5.5,2.5,4.0,1.3],
	[5.5,2.6,4.4,1.2],
	[6.1,3.0,4.6,1.4],
	[5.8,2.6,4.0,1.2],
	[5.0,2.3,3.3,1.0],
	[5.6,2.7,4.2,1.3],
	[5.7,3.0,4.2,1.2],
	[5.7,2.9,4.2,1.3],
	[6.2,2.9,4.3,1.3],
	[5.1,2.5,3.0,1.1],
	[5.7,2.8,4.1,1.3],
	[6.3,3.3,6.0,2.5],
	[5.8,2.7,5.1,1.9],
	[7.1,3.0,5.9,2.1],
	[6.3,2.9,5.6,1.8],
	[6.5,3.0,5.8,2.2],
	[7.6,3.0,6.6,2.1],
	[4.9,2.5,4.5,1.7],
	[7.3,2.9,6.3,1.8],
	[6.7,2.5,5.8,1.8],
	[7.2,3.6,6.1,2.5],
	[6.5,3.2,5.1,2.0],
	[6.4,2.7,5.3,1.9],
	[6.8,3.0,5.5,2.1],
	[5.7,2.5,5.0,2.0],
	[5.8,2.8,5.1,2.4],
	[6.4,3.2,5.3,2.3],
	[6.5,3.0,5.5,1.8],
	[7.7,3.8,6.7,2.2],
	[7.7,2.6,6.9,2.3],
	[6.0,2.2,5.0,1.5],
	[6.9,3.2,5.7,2.3],
	[5.6,2.8,4.9,2.0],
	[7.7,2.8,6.7,2.0],
	[6.3,2.7,4.9,1.8],
	[6.7,3.3,5.7,2.1],
	[7.2,3.2,6.0,1.8],
	[6.2,2.8,4.8,1.8],
	[6.1,3.0,4.9,1.8],
	[6.4,2.8,5.6,2.1],
	[7.2,3.0,5.8,1.6],
	[7.4,2.8,6.1,1.9],
	[7.9,3.8,6.4,2.0],
	[6.4,2.8,5.6,2.2],
	[6.3,2.8,5.1,1.5],
	[6.1,2.6,5.6,1.4],
	[7.7,3.0,6.1,2.3],
	[6.3,3.4,5.6,2.4],
	[6.4,3.1,5.5,1.8],
	[6.0,3.0,4.8,1.8],
	[6.9,3.1,5.4,2.1],
	[6.7,3.1,5.6,2.4],
	[6.9,3.1,5.1,2.3],
	[5.8,2.7,5.1,1.9],
	[6.8,3.2,5.9,2.3],
	[6.7,3.3,5.7,2.5],
	[6.7,3.0,5.2,2.3],
	[6.3,2.5,5.0,1.9],
	[6.5,3.0,5.2,2.0],
	[6.2,3.4,5.4,2.3],
	[5.9,3.0,5.1,1.8]
]; // list of inputs (n, w, h)
const Y = [ // list of label, one-hot encoding (n, j)
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[1, 0, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 1, 0],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1],
	[0, 0, 1]
]; // list of label, one-hot encoding (n, j)