ActivationFunction class for Neural Network

activation_function.js

class ActivationFunction{
	constructor(activator,deactivator,usePrev){
		this.activator = activator;
		this.deactivator = deactivator;
		this.usePrev = usePrev;
	}

	static get SIGMOID(){
		return new ActivationFunction(value => 1 / (1 + Math.exp(value)),value => value * (1 - value),false);
	}
	static get TANH(){
		return new ActivationFunction(value => Math.tanh(value),value => 1 - value * value,false);
	}
	static get ARCTAN(){
		return new ActivationFunction(value => Math.atan(value),value => 1 / (value * value) + 1,true);
	}
	static get SOFTSIGN(){
		return new ActivationFunction(value => value / (1 + Math.abs(value)),value => 1 / Math.pow((Math.abs(value) + 1),2),true);
	}
	static get RELU(){
		return new ActivationFunction(value => value < 0 ? 0 : value,value => value < 0 ? 0 : 1,true);
	}
	static get LEAKY_RELU(){
		return new ActivationFunction(value => value < 0 ? 0.01 * value : value,value => value < 0 ? 0.01 : 1,true);
	}
	static get SOFT_PLUS(){
		return new ActivationFunction(value => Math.log(1 + Math.exp(value)),value => 1 / (1 + Math.exp(-value)),true);
	}
	static get GAUSSIAN(){
		return new ActivationFunction(value => Math.exp(-value * value),value => -2 * value * Math.exp(-value * value),true	);
	}
}

nn.js

class NeuralNetwork {
    constructor(...nodes) { 
        this.weights = [];
        this.biases = [];
        this.layers = nodes.length-1;

        this.activation_function = ActivationFunction.SIGMOID;

        nodes.forEach((node,i,arr) => {
            if(i === 0) return;
            let weight = new Matrix(node,arr[i-1]).randomize();
            this.weights.push(weight);
            let bias = new Matrix(node,1).randomize();
            this.biases.push(bias);
        });

        this.learning_rate = 0.1;
    }

    setActivator(type){
        if(ActivationFunction[type] === undefined) throw new Error(`ActivationFunction.${type} is undefined`);
        this.activation_function = ActivationFunction[type];
    }

    predict(input_array) {
        // Convert input into Matrix
        let input = Matrix.fromArray(input_array);

        // Calculate output
        let output;
        this.weights.forEach((weight,i,arr) => {
            output = Matrix.multiply(weight,(i === 0) ? input : output);
            output.add(this.biases[i]);
            output.map(this.activation_function.activator);
        });

        // Sending back to the caller!
        return output.toArray();
    }

    train(input_array, target_array) {
        // Generating the Hidden Outputs
        let inputs = Matrix.fromArray(input_array);
        let targets = Matrix.fromArray(target_array);

        // Store all layers output
        let unActivatedOutputs = [];
        let outputs = [];
        this.weights.forEach((weight,i,arr) => {
            let output = Matrix.multiply(weight,(i === 0) ? inputs : outputs[i-1]);
            output.add(this.biases[i]);
            unActivatedOutputs.push(output);
            output.map(this.activation_function.activator);
            outputs.push(output);
        });

        // Backpropagate and adjust the weights and biases
        let error = Matrix.subtract(targets,outputs[outputs.length-1]);
        for(let i=outputs.length-1;i>=0;i--){
            if(i < outputs.length-1){
                error = Matrix.multiply(Matrix.transpose(this.weights[i+1]),error);
            }

            let gradient = Matrix.map(this.activation_function.usePrev ? unActivatedOutputs[i] : outputs[i],this.activation_function.deactivator);
            gradient.multiply(error);
            gradient.multiply(this.learning_rate);

            let delta = Matrix.multiply(gradient,Matrix.transpose((i === 0) ? inputs : outputs[i-1]));

            this.weights[i].add(delta);
            this.biases[i].add(gradient);
        }
    }

    print(){
        this.weights.forEach((weight,i) => {
            let wTable = weight.html();
            let bTable = this.biases[i].html();
            document.body.innerHTML += "<div class='side'>" + wTable + bTable + "</div><br>";
        });
        document.body.innerHTML += "<hr>";
    }
}