Skip to content

Instantly share code, notes, and snippets.

@adrianseeley
Last active November 9, 2017 06:43
Show Gist options
  • Save adrianseeley/a19d6721f79e2082fe06e32a5c3a0ed2 to your computer and use it in GitHub Desktop.
Save adrianseeley/a19d6721f79e2082fe06e32a5c3a0ed2 to your computer and use it in GitHub Desktop.
Neural Network with Backpropagation, Bootstrapping and Bagging (Uses MSE)
function mse (errors) {
var sum = 0;
for (var i = 0; i < errors.length; i++) {
sum += Math.pow(errors[i], 2);
}
return sum / errors.length;
};
function zero_array (size) {
var ary = [];
for (var s = 0; s < size; s++) {
ary.push(0);
}
return ary;
};
function random_array_range (size, low, high) {
var range = high - low;
var ary = [];
for (var s = 0; s < size; s++) {
ary.push((Math.random() * range) + low);
}
return ary;
};
function delta_abs_ary (a, b) {
var d = zero_array(a.length);
for (var i = 0; i < b.length; b++) {
d[i] = Math.abs(b[i] - a[i]);
}
return d;
};
function vector_class_match (a, b) {
var class_a = vector_class(a);
var class_b = vector_class(b);
if (class_a == class_b) {
return 1;
} else {
return 0;
}
};
function vector_class (a) {
var max_component = -1;
var max_component_value = -Infinity;
for (var i = 0; i < a.length; i++) {
if (a[i] > max_component_value) {
max_component = i;
max_component_value = a[i];
}
}
return max_component;
};
function neural_network (layer_sizes) {
var nn = {
layer_sizes: layer_sizes,
output_layer: layer_sizes.length - 1,
biases: [],
weights: [],
outputs: [],
deltas: [],
changes: [],
errors: [],
};
nn.run = function (input) {
nn.outputs[0] = input;
for (var layer = 1; layer <= nn.output_layer; layer++) {
for (var node = 0; node < nn.layer_sizes[layer]; node++) {
// linear weighting
var weights = nn.weights[layer - 1][node];
var sum = nn.biases[layer - 1][node];
for (var i = 0; i < weights.length; i++) {
sum += weights[i] * input[i];
}
// activation function
nn.outputs[layer][node] = 1 / (1 + Math.exp(-sum));
}
input = nn.outputs[layer];
}
return input;
};
nn.errors_fitness = function (data_rows) {
var errors = [];
for (var r = 0; r < data_rows.length; r++) {
var estimate = nn.run(data_rows[r][0]);
var actual = data_rows[r][1];
var delta = delta_abs_ary(estimate, actual);
errors.push(delta);
}
return mse(errors);
};
nn.errors_classify = function (data_rows) {
var correct = 0;
for (var r = 0; r < data_rows.length; r++) {
var estimate = nn.run(data_rows[r][0]);
var actual = data_rows[r][1];
correct += vector_class_match(estimate, actual);
}
return 1.0 - (correct / data_rows.length);
};
nn.test = function (data_rows) {
var outputs = [];
for (var r = 0; r < data_rows.length; r++) {
var estimate = nn.run(data_rows[r][0]);
outputs.push(estimate);
}
return outputs;
};
nn.test_classify = function (data_rows) {
var outputs = [];
for (var r = 0; r < data_rows.length; r++) {
var estimate = nn.run(data_rows[r][0]);
outputs.push(vector_class(estimate));
}
return outputs;
};
nn.train = function (data_rows, cfg) {
cfg.iterations = cfg.iterations || 20000;
cfg.error_threshold = cfg.error_threshold || 0.005;
cfg.learning_rate = cfg.learning_rate || 0.3;
cfg.verbose = cfg.verbose || true;
cfg.verbose_log_period = cfg.verbose_log_period || 10;
var last_error = null;
var error = 1;
var iteration = 0;
for (iteration = 0; iteration < cfg.iterations && error > cfg.error_threshold && error != last_error; iteration++) {
last_error = error;
var sum = 0;
for (var r = 0; r < data_rows.length; r++) {
sum += nn.train_pattern(data_rows[r][0], data_rows[r][1], cfg);
}
error = sum / data_rows.length;
if (cfg.verbose && (iteration % cfg.verbose_log_period == 0)) {
console.log((cfg.verbose_network_index != undefined ? 'network: ' + cfg.verbose_network_index + '/' + cfg.verbose_network_count + ', ' : '') + 'iterations: ' + iteration + '/' + cfg.iterations +', training error: ' + error + '/' + cfg.error_threshold);
}
}
return {
error: error,
iterations: iteration
};
};
nn.train_pattern = function (input, output, cfg) {
nn.run(input);
nn.calculate_deltas(output);
nn.adjust_weights(cfg.learning_rate, cfg.momentum);
return mse(nn.errors[nn.output_layer]);
};
nn.calculate_deltas = function (output) {
for (var layer = nn.output_layer; layer >= 0; layer--) {
for (var node = 0; node < nn.layer_sizes[layer]; node++) {
var nn_output = nn.outputs[layer][node];
var error = 0;
if (layer == nn.output_layer) {
error = output[node] - nn_output;
}
else {
var deltas = nn.deltas[layer + 1];
for (var i = 0; i < deltas.length; i++) {
error += deltas[i] * nn.weights[layer][i][node];
}
}
nn.errors[layer][node] = error;
nn.deltas[layer][node] = error * nn_output * (1 - nn_output);
}
}
};
nn.adjust_weights = function (learning_rate, momentum) {
for (var layer = 1; layer <= nn.output_layer; layer++) {
var incoming = nn.outputs[layer - 1];
for (var node = 0; node < nn.layer_sizes[layer]; node++) {
var delta = nn.deltas[layer][node];
for (var i = 0; i < incoming.length; i++) {
var change = nn.changes[layer - 1][node][i];
change = (learning_rate * delta * incoming[i]) + (momentum * change);
nn.changes[layer - 1][node][i] = change;
nn.weights[layer - 1][node][i] += change;
}
nn.biases[layer - 1][node] += learning_rate * delta;
}
}
};
for (var layer = 0; layer <= nn.output_layer; layer++) {
var layer_size = nn.layer_sizes[layer];
nn.deltas.push(zero_array(layer_size));
nn.errors.push(zero_array(layer_size));
nn.outputs.push(zero_array(layer_size));
if (layer > 0) {
nn.biases.push(random_array_range(layer_size, -0.2, 0.2));
nn.weights.push([]);
nn.changes.push([]);
for (var node = 0; node < layer_size; node++) {
var previous_layer_size = nn.layer_sizes[layer - 1];
nn.weights[nn.weights.length - 1].push(random_array_range(previous_layer_size, -0.2, 0.2));
nn.changes[nn.changes.length - 1].push(zero_array(previous_layer_size));
}
}
}
return nn;
};
function weight_drop_neural_network (layer_sizes) {
var wdnn = neural_network(layer_sizes);
wdnn.run_with_weight_drop = function (input, drop_rate) {
wdnn.outputs[0] = input;
for (var layer = 1; layer <= wdnn.output_layer; layer++) {
for (var node = 0; node < wdnn.layer_sizes[layer]; node++) {
// linear weighting
var weights = wdnn.weights[layer - 1][node];
var sum = wdnn.biases[layer - 1][node];
for (var i = 0; i < weights.length; i++) {
sum += (weights[i] * input[i]) * (Math.random() < drop_rate ? 0 : 1);
}
// activation function
wdnn.outputs[layer][node] = 1 / (1 + Math.exp(-sum));
}
input = wdnn.outputs[layer];
}
return input;
};
wdnn.train_pattern = function (input, output, cfg) {
wdnn.run_with_weight_drop(input, cfg.drop_rate);
wdnn.calculate_deltas(output);
wdnn.adjust_weights(cfg.learning_rate, cfg.momentum);
return mse(wdnn.errors[wdnn.output_layer]);
};
return wdnn;
};
function bagged_neural_network (neural_network_count, layer_sizes, fn_neural_network) {
var bnn = {
neural_network_count: neural_network_count,
layer_sizes: layer_sizes,
output_layer: layer_sizes.length - 1,
networks: []
};
for (var n = 0; n < neural_network_count; n++) {
bnn.networks.push(fn_neural_network(layer_sizes));
}
bnn.run = function (input) {
var output_sum = zero_array(bnn.layer_sizes[bnn.output_layer]);
for (var n = 0; n < bnn.networks.length; n++) {
var output = bnn.networks[n].run(input);
for (var o = 0; o < output.length; o++) {
output_sum[o] += output[o];
}
}
for (var o = 0; o < output_sum.length; o++) {
output_sum[o] /= bnn.neural_network_count;
}
return output_sum;
};
bnn.errors_fitness = function (data_rows) {
var errors = [];
for (var r = 0; r < data_rows.length; r++) {
var estimate = bnn.run(data_rows[r][0]);
var actual = data_rows[r][1];
var delta = delta_abs_ary(estimate, actual);
errors.push(delta);
}
return mse(errors);
};
bnn.errors_classify = function (data_rows) {
var correct = 0;
for (var r = 0; r < data_rows.length; r++) {
var estimate = bnn.run(data_rows[r][0]);
var actual = data_rows[r][1];
correct += vector_class_match(estimate, actual);
}
return 1.0 - (correct / data_rows.length);
};
bnn.test = function (data_rows) {
var outputs = [];
for (var r = 0; r < data_rows.length; r++) {
var estimate = bnn.run(data_rows[r][0]);
outputs.push(estimate);
}
return outputs;
};
bnn.test_classify = function (data_rows) {
var outputs = [];
for (var r = 0; r < data_rows.length; r++) {
var estimate = bnn.run(data_rows[r][0]);
outputs.push(vector_class(estimate));
}
return outputs;
};
bnn.train = function (data_rows, cfg) {
var errors = [];
cfg.verbose_network_count = bnn.networks.length;
for (var n = 0; n < bnn.networks.length; n++) {
cfg.verbose_network_index = n;
errors.push(bnn.networks[n].train(data_rows, cfg));
}
return errors;
};
return bnn
};
function bootstrapped_neural_network (neural_network_count, layer_sizes, fn_neural_network) {
var bnn = bagged_neural_network(neural_network_count, layer_sizes, fn_neural_network);
bnn.train = function (data_rows, cfg) {
var errors = [];
cfg.verbose_network_count = bnn.networks.length;
for (var n = 0; n < bnn.networks.length; n++) {
var samples = [];
cfg.verbose_network_index = n;
for (var s = 0; s < cfg.samples_per_network; s++) {
samples.push(data_rows[Math.floor(Math.random() * data_rows.length)]);
}
errors.push(bnn.networks[n].train(samples, cfg));
}
return errors;
};
return bnn
};
var ocr_train = require('./data/ocr/train.js');
var ocr_test = require('./data/ocr/test.js');
var ocr_write_results = require('./data/ocr/write_results.js');
console.log(ocr_train[0][0].length, ocr_train[0][1].length);
console.log(ocr_test[0][0].length, ocr_test.length);
var bnn = bootstrapped_neural_network(10, [784, 100, 10], weight_drop_neural_network);
var cfg_train = {
drop_rate: 0.5,
samples_per_network: 500,
iterations: 100,
error_threshold: 0.005,
learning_rate: 0.3,
momentum: 0.1,
verbose: true,
verbose_log_period: 5
};
var errors = bnn.train(ocr_train, cfg_train);
console.log('network errors', errors);
console.log('accuracy', bnn.errors_classify(ocr_train));
var test_answers = bnn.test_classify(ocr_test);
ocr_write_results(test_answers);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment