Skip to content

Instantly share code, notes, and snippets.

@srifqi
Created May 10, 2023 02:00
Show Gist options
  • Save srifqi/a9e752f87a5ddf2031dfcff21b337e27 to your computer and use it in GitHub Desktop.
Save srifqi/a9e752f87a5ddf2031dfcff21b337e27 to your computer and use it in GitHub Desktop.
An implementation example of multilayer perceptron (MLP) in pure ES6 JavaScript without third-party libraries
/**
* A program to create, train, and use a simple MLP
*
* Below is an implementation example of multilayer perceptron (MLP) in pure ES6
* JavaScript without third party libraries.
*
* @author srifqi
* @license MIT License
*/
/** ======= MAIN COMPONENTS' FUNCTIONS ======= **/
/**
* @typedef MLP
* @type {object}
* @property {number[][]} layers Interlayer weight matrix
* @property {number[][]} bias Bias weight matrix for each layer (after input layer)
* @property {function[]} activations List of activation functions of each layer (after input layer)
* @property {function[]} derActivations List of activation functions' (first) derivation of each layer (after input layer)
*/
/**
* Main function to create an MLP
*
* @param {number[]} sizes List of sizes for each layer (including input layer)
* @param {number[]} activations List of activation functions for each layer (after input layer)
* @returns {MLP} An MLP model with the given parameters
*/
function createMLP(sizes, activations) {
const model = {
layers: [],
bias: [],
activations: [],
derActivations: []
};
for (let i = 1; i < sizes.length; i ++) {
const sA = sizes[i - 1];
const sB = sizes[i];
model.layers.push(createRandom2DMatrix(sB, sA));
model.bias.push(createRandom2DMatrix(sB, 1));
}
for (let i = 0; i < activations.length; i ++) {
const ai = activations[i];
model.activations.push(listGenActivations[ai]);
model.derActivations.push(listDerActivations[ai]);
}
return model;
}
/**
* Main function to train an MLP
*
* The training is done stochastically, i.e. the parameter update is done for
* each entry and not using its average error.
*
* @param {MLP} model Model to be trained
* @param {number[][]} X List of inputs from the training dataset
* @param {number[][]} Y List of outputs (results/targets) from the training dataset
* @param {Function} learningRateFunction Function to calculate learning rate constant
* @param {number} maxIter Amount of maximum iteration
* @returns {MLP} Trained model
*/
function train(model, X, Y, learningRateFunction, maxIter) {
if (X.length != Y.length) {
throw new Error(`The size of X and Y doesn't match: ${X.length} with ${Y.length}.`);
}
let lastLearningRate = 0;
// each iteration
for (let iter = 1; iter <= maxIter; iter ++) {
const learningRate = learningRateFunction(iter - 1);
if (lastLearningRate != learningRate) {
console.log(`Learning rate is set to ${learningRate}.`);
lastLearningRate = learningRate;
}
let correct = 0;
let totalError = 0;
// each entry
for (let k = 0; k < X.length; k ++) {
// feed-forward
const inputs = transpose([X[k]]);
const listOfValues = feedForward(model, inputs);
const outputs = listOfValues[listOfValues.length - 1];
// to calculate accuracy
if (argMax(transpose(outputs)[0]) == argMax(Y[k])) {
correct ++;
}
// back-propagation
let error = substract(outputs, transpose([Y[k]]));
totalError += transpose(error)[0].reduce((a, v) => a + v);
// no multiplication with output layer's activation function's derivation
for (let i = model.layers.length - 1; i >= 0; i --) {
const layerError = multiply(
error,
transpose(listOfValues[i]),
learningRate
);
const layerBiasError = multiply(
error,
learningRate
)
model.layers[i] = substract(model.layers[i], layerError);
model.bias[i] = substract(model.bias[i], layerBiasError);
if (i > 0) {
error = multiply(transpose(model.layers[i]), model.derActivations[i - 1](error));
}
}
}
const accuracy = correct / X.length;
const avgError = totalError / X.length;
console.log(`Iteration ${iter}/${maxIter} | Error: ${avgError} | Accuracy: ${correct}/${X.length} / ${accuracy}`);
}
return model;
}
/**
* Main function to predict using given MLP
*
* @param {MLP} model Model that will predict
* @param {number[][]} X List of inputs
* @returns {Object.<string,*>}
*/
function predict(model, X) {
const result = {
outputs: [],
selected: []
}
for (let k = 0; k < X.length; k ++) {
const inputs = transpose([X[k]]);
const listOfValues = feedForward(model, inputs);
const outputs = listOfValues[listOfValues.length - 1];
const selected = argMax(transpose(outputs)[0]);
result.outputs.push(outputs);
result.selected.push(selected);
}
return result;
}
/** ======= MATHEMATICS HELPER FUNCTIONS ======= **/
/**
* Helper function to create a 2 dimensional matrix with a uniform fill
*
* @param {number} rows Amount of rows
* @param {number} columns Amount columns
* @param {number} [values=0] Value for each matrix's element
* @returns {number} New matrix using given parameters
*/
function create2DMatrix(rows, columns, values) {
if (values == undefined) {
values = 0;
}
return Array.from(Array(rows), () => Array.from(Array(columns), () => values));
}
/**
* Helper function to create a 2 dimensional matrix with a random fill
*
* @param {number} rows Amount of rows
* @param {number} columns Amount columns
* @returns {number} New matrix using given parameters
*/
function createRandom2DMatrix(rows, columns) {
return Array.from(Array(rows), () => Array.from(Array(columns), () => Math.random()));
}
/**
* Helper function to transpose a matrix
*
* @param {number[][]} matrix Matrix to be transposed
* @returns Transposed matrix
*/
function transpose(matrix) {
let result = create2DMatrix(matrix[0].length, matrix.length);
for (let i = 0; i < matrix.length; i ++) {
for (let j = 0; j < matrix[0].length; j ++) {
result[j][i] = matrix[i][j];
}
}
return result;
}
/**
* Helper function to multiply many values
*
* Given values can be a matrix or a scalar.
*
* @param {...number|number[][]} elements List of values to be multiplied
* @returns {number|number[][]} Result of multiplication
*/
function multiply(...elements) {
if (elements.length == undefined || elements.length <= 1) {
throw new Error("There is only one value that will be multiplied.");
}
let result = elements[0];
for (let q = 1; q < elements.length; q ++) {
let ki = result;
let ka = elements[q];
// both are scalars
if (ki.length == undefined && ka.length == undefined) {
result = ki * ka;
// both are matrices
} else if (ki.length >= 1 && ka.length >= 1) {
if (ki[0].length != ka.length) {
let kiy = ki.length;
let kix = ki[0].length;
let kay = ka.length;
let kax = ka[0].length;
throw new Error(`Matrices' size doesn't match: ${kiy}x${kix} with ${kay}x${kax}.`);
}
result = create2DMatrix(ki.length, ka[0].length);
for (let i = 0; i < ki.length; i ++) {
for (let j = 0; j < ka[0].length; j ++) {
let total = 0;
for (let k = 0; k < ka.length; k ++) {
total += ki[i][k] * ka[k][j];
}
result[i][j] = total;
}
}
// one of those is a matrix
} else {
let scalar = ki;
let matrix = ka;
if (ka.length == undefined) { // only check one of those
scalar = ka;
matrix = ki;
}
for (let i = 0; i < matrix.length; i ++) {
for (let j = 0; j < matrix[0].length; j ++) {
matrix[i][j] *= scalar;
}
}
result = matrix;
}
}
return result;
}
/**
* Helper function to add two matrices
*
* Both matrices need to have the same size.
*
* @param {number[][]} A Matrix A
* @param {number[][]} B Matrix B
* @returns {number[][]} Result of addition between two matrices: A + B.
*/
function addition(A, B) {
if (A.length != B.length || A[0].length != B[0].length) {
let Ay = A.length;
let Ax = A[0].length;
let By = B.length;
let Bx = B[0].length;
throw new Error(`Matrices' size are not the same: ${Ay}x${Ax} with ${By}x${Bx}.`);
}
for (let i = 0; i < A.length; i ++) {
for (let j = 0; j < A[0].length; j ++) {
A[i][j] += B[i][j];
}
}
return A;
}
/**
* Helper function to substract two matrices
*
* Both matrices need to have the same size.
*
* @param {number[][]} A Matrix A
* @param {number[][]} B Matrix B
* @returns {number[][]} Result of substraction from matrix A by matrix B: A - B.
*/
function substract(A, B) {
if (A.length != B.length || A[0].length != B[0].length) {
let Ay = A.length;
let Ax = A[0].length;
let By = B.length;
let Bx = B[0].length;
throw new Error(`Matrices' size are not the same: ${Ay}x${Ax} with ${By}x${Bx}.`);
}
for (let i = 0; i < A.length; i ++) {
for (let j = 0; j < A[0].length; j ++) {
A[i][j] -= B[i][j];
}
}
return A;
}
/**
* Helper function to find an index which has the maximum value
*
* If there are many maximum values, the lowest index is chosen.
*
* @param {number[]} list List that will be searched
* @returns {number} Index which has the maximum value
*/
function argMax(list) {
if (list.length == undefined) {
throw new Error("Input is not a list.");
}
if (list.length == 0) {
throw new Error("Given list is empty.");
}
let maxIndex = 0;
for (let q = 1; q < list.length; q ++) {
if (list[q] > list[maxIndex]) {
maxIndex = q;
}
}
return maxIndex;
}
/**
* Helper function to do feed-forward
*
* @param {MLP} model Model which will be used
* @param {number[][]} X List of inputs
* @returns {number[][]} List of outputs for each layer (after input layer)
*/
function feedForward(model, X) {
let values = X;
let listOfResults = [values];
for (let i = 0; i < model.layers.length; i ++) {
values = multiply(model.layers[i], values);
values = addition(values, model.bias[i]);
values = model.activations[i](values);
listOfResults.push(values);
}
return listOfResults;
}
/**
* List of activation functions
*
* @type {Object<string,Function>}
*/
let listGenActivations = {};
/**
* List of activation functions' derivation
*
* @type {Object<string,Function>}
*/
let listDerActivations = {};
/**
* Helper function to apply ReLU for each element in a matrix
*
* @param {number[][]} x Input matrix
* @returns {number[][]} Resulting matrix after ReLU mapping
*/
function genReLU(x) {
for (let i = 0; i < x.length; i ++) {
for (let j = 0; j < x[i].length; j ++) {
x[i][j] = Math.max(x[i][j], 0);
}
}
return x;
}
listGenActivations["relu"] = genReLU;
/**
* Helper function to apply ReLU's derivation for each element in a matrix
*
* @param {number[][]} x Input matrix
* @returns {number[][]} Resulting matrix after ReLU's derivation mapping
*/
function derReLU(x) {
for (let i = 0; i < x.length; i ++) {
for (let j = 0; j < x[i].length; j ++) {
x[i][j] = x[i][j] >= 0 ? 1 : 0;
}
}
return x;
}
listDerActivations["relu"] = derReLU;
/** ======= MAIN PROGRAM ======= **/
/**
* Main function that will be run first (see last section)
*/
function main() {
const sizes = [4, 5, 3];
const activations = ["relu", "relu"]; // only ReLU
const model = createMLP(sizes, activations);
const constantLearningRate = () => 0.001;
train(model, X, Y, constantLearningRate, 4);
console.log(predict(model, [X[0]])); // prediction example
}
/** ======= PROGRAM EXECUTION ======= **/
// Example of Iris dataset (see below)
const X = [ // list of inputs (n, w, h)
[5.1, 3.5, 1.4, 0.2]
// ...
]; // list of inputs (n, w, h)
const Y = [ // list of label, one-hot encoding (n, j)
[1, 0, 0]
// ...
]; // list of label, one-hot encoding (n, j)
main();
const X = [ // list of inputs (n, w, h)
[5.1,3.5,1.4,0.2],
[4.9,3.0,1.4,0.2],
[4.7,3.2,1.3,0.2],
[4.6,3.1,1.5,0.2],
[5.0,3.6,1.4,0.2],
[5.4,3.9,1.7,0.4],
[4.6,3.4,1.4,0.3],
[5.0,3.4,1.5,0.2],
[4.4,2.9,1.4,0.2],
[4.9,3.1,1.5,0.1],
[5.4,3.7,1.5,0.2],
[4.8,3.4,1.6,0.2],
[4.8,3.0,1.4,0.1],
[4.3,3.0,1.1,0.1],
[5.8,4.0,1.2,0.2],
[5.7,4.4,1.5,0.4],
[5.4,3.9,1.3,0.4],
[5.1,3.5,1.4,0.3],
[5.7,3.8,1.7,0.3],
[5.1,3.8,1.5,0.3],
[5.4,3.4,1.7,0.2],
[5.1,3.7,1.5,0.4],
[4.6,3.6,1.0,0.2],
[5.1,3.3,1.7,0.5],
[4.8,3.4,1.9,0.2],
[5.0,3.0,1.6,0.2],
[5.0,3.4,1.6,0.4],
[5.2,3.5,1.5,0.2],
[5.2,3.4,1.4,0.2],
[4.7,3.2,1.6,0.2],
[4.8,3.1,1.6,0.2],
[5.4,3.4,1.5,0.4],
[5.2,4.1,1.5,0.1],
[5.5,4.2,1.4,0.2],
[4.9,3.1,1.5,0.1],
[5.0,3.2,1.2,0.2],
[5.5,3.5,1.3,0.2],
[4.9,3.1,1.5,0.1],
[4.4,3.0,1.3,0.2],
[5.1,3.4,1.5,0.2],
[5.0,3.5,1.3,0.3],
[4.5,2.3,1.3,0.3],
[4.4,3.2,1.3,0.2],
[5.0,3.5,1.6,0.6],
[5.1,3.8,1.9,0.4],
[4.8,3.0,1.4,0.3],
[5.1,3.8,1.6,0.2],
[4.6,3.2,1.4,0.2],
[5.3,3.7,1.5,0.2],
[5.0,3.3,1.4,0.2],
[7.0,3.2,4.7,1.4],
[6.4,3.2,4.5,1.5],
[6.9,3.1,4.9,1.5],
[5.5,2.3,4.0,1.3],
[6.5,2.8,4.6,1.5],
[5.7,2.8,4.5,1.3],
[6.3,3.3,4.7,1.6],
[4.9,2.4,3.3,1.0],
[6.6,2.9,4.6,1.3],
[5.2,2.7,3.9,1.4],
[5.0,2.0,3.5,1.0],
[5.9,3.0,4.2,1.5],
[6.0,2.2,4.0,1.0],
[6.1,2.9,4.7,1.4],
[5.6,2.9,3.6,1.3],
[6.7,3.1,4.4,1.4],
[5.6,3.0,4.5,1.5],
[5.8,2.7,4.1,1.0],
[6.2,2.2,4.5,1.5],
[5.6,2.5,3.9,1.1],
[5.9,3.2,4.8,1.8],
[6.1,2.8,4.0,1.3],
[6.3,2.5,4.9,1.5],
[6.1,2.8,4.7,1.2],
[6.4,2.9,4.3,1.3],
[6.6,3.0,4.4,1.4],
[6.8,2.8,4.8,1.4],
[6.7,3.0,5.0,1.7],
[6.0,2.9,4.5,1.5],
[5.7,2.6,3.5,1.0],
[5.5,2.4,3.8,1.1],
[5.5,2.4,3.7,1.0],
[5.8,2.7,3.9,1.2],
[6.0,2.7,5.1,1.6],
[5.4,3.0,4.5,1.5],
[6.0,3.4,4.5,1.6],
[6.7,3.1,4.7,1.5],
[6.3,2.3,4.4,1.3],
[5.6,3.0,4.1,1.3],
[5.5,2.5,4.0,1.3],
[5.5,2.6,4.4,1.2],
[6.1,3.0,4.6,1.4],
[5.8,2.6,4.0,1.2],
[5.0,2.3,3.3,1.0],
[5.6,2.7,4.2,1.3],
[5.7,3.0,4.2,1.2],
[5.7,2.9,4.2,1.3],
[6.2,2.9,4.3,1.3],
[5.1,2.5,3.0,1.1],
[5.7,2.8,4.1,1.3],
[6.3,3.3,6.0,2.5],
[5.8,2.7,5.1,1.9],
[7.1,3.0,5.9,2.1],
[6.3,2.9,5.6,1.8],
[6.5,3.0,5.8,2.2],
[7.6,3.0,6.6,2.1],
[4.9,2.5,4.5,1.7],
[7.3,2.9,6.3,1.8],
[6.7,2.5,5.8,1.8],
[7.2,3.6,6.1,2.5],
[6.5,3.2,5.1,2.0],
[6.4,2.7,5.3,1.9],
[6.8,3.0,5.5,2.1],
[5.7,2.5,5.0,2.0],
[5.8,2.8,5.1,2.4],
[6.4,3.2,5.3,2.3],
[6.5,3.0,5.5,1.8],
[7.7,3.8,6.7,2.2],
[7.7,2.6,6.9,2.3],
[6.0,2.2,5.0,1.5],
[6.9,3.2,5.7,2.3],
[5.6,2.8,4.9,2.0],
[7.7,2.8,6.7,2.0],
[6.3,2.7,4.9,1.8],
[6.7,3.3,5.7,2.1],
[7.2,3.2,6.0,1.8],
[6.2,2.8,4.8,1.8],
[6.1,3.0,4.9,1.8],
[6.4,2.8,5.6,2.1],
[7.2,3.0,5.8,1.6],
[7.4,2.8,6.1,1.9],
[7.9,3.8,6.4,2.0],
[6.4,2.8,5.6,2.2],
[6.3,2.8,5.1,1.5],
[6.1,2.6,5.6,1.4],
[7.7,3.0,6.1,2.3],
[6.3,3.4,5.6,2.4],
[6.4,3.1,5.5,1.8],
[6.0,3.0,4.8,1.8],
[6.9,3.1,5.4,2.1],
[6.7,3.1,5.6,2.4],
[6.9,3.1,5.1,2.3],
[5.8,2.7,5.1,1.9],
[6.8,3.2,5.9,2.3],
[6.7,3.3,5.7,2.5],
[6.7,3.0,5.2,2.3],
[6.3,2.5,5.0,1.9],
[6.5,3.0,5.2,2.0],
[6.2,3.4,5.4,2.3],
[5.9,3.0,5.1,1.8]
]; // list of inputs (n, w, h)
const Y = [ // list of label, one-hot encoding (n, j)
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 1, 0],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1],
[0, 0, 1]
]; // list of label, one-hot encoding (n, j)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment