N-layer ANN

ann.c

#include <stdio.h>
#include <malloc.h>
#include <math.h>
#include <stdlib.h>

//
// Generic N-layer neuron network code (using unipolar sigmoid activation function)
//

// Data type to operate on; typically double.
typedef double data_t;
// Synaptic weights type.
typedef double weight_t;

// Defines a single network layer properties
// Not used by user code
typedef struct _layer_t {
    // Number of inputs for each node
    size_t input_count;
    // Number of nodes on this layer = number of outputs
    size_t output_count;
    // Pointer to weights matrix; size = input_count * output_count
    weight_t *weights;
} layer_t;

void layer_randomise_weights(weight_t *weights, size_t count)
{
    weight_t *limit = weights + count;
    while (weights < limit) {
        *weights++ = (2.0 / RAND_MAX) * (rand() - RAND_MAX / 2);
    }
}

void layer_propagate_forward(const layer_t *layer, const data_t *inputs, data_t *outputs)
{
    size_t n, i;
    const weight_t *weights = layer->weights;

    // Multiply input vector by weight matrix
    for (n = 0; n < layer->output_count; ++n) {
        data_t accum = 0.0;
        for (i = 0; i < layer->input_count; ++i) {
            accum += inputs[i] * (*weights++);
        }
        outputs[n] = 1.0 / (1.0 + exp(-accum));
    }
}

void layer_propagate_errors(const layer_t *layer, const data_t *inputs, const data_t *outputs, const data_t *error_inputs, data_t *error_outputs)
{
    size_t n, i;
    data_t error;
    weight_t weight;

    for (i = 0; i < layer->input_count; ++i) {
        error_outputs[i] = 0.0;
    }

    for (n = 0; n < layer->output_count; ++n) {
        // Node error value
        error = outputs[n] * (1.0 - outputs[n]) * error_inputs[n];
        for (i = 0; i < layer->input_count; ++i) {
            // Save old weight
            weight = layer->weights[n * layer->input_count + i];
            // Update weight
            layer->weights[n * layer->input_count + i] = weight + error * inputs[i];
            // Propagate the error with old weight
            error_outputs[i] += error * weight;
        }
    }
}

// Defines the neuron network
typedef struct _network_t {
    // Number of layers
    size_t layer_count;
    // Layer data
    layer_t *layers;
    // Internal layer states; count = layer_count + 1
    // states[0] = pointer to inputs (not stored)
    // states[layer_count] = pointer to outputs (not stored)
    data_t **states;
    data_t **errors;
} network_t;

network_t *network_create(size_t layer_count, const size_t node_counts[])
{
    network_t *net;
    size_t i;

    net = (network_t *)malloc(sizeof(network_t));
    net->layer_count = layer_count;

    net->layers = (layer_t *)malloc(sizeof(layer_t) * layer_count);
    for (i = 0; i < layer_count; ++i) {
        layer_t *layer = &net->layers[i];
        size_t weight_count;

        layer->input_count = node_counts[i];
        layer->output_count = node_counts[i + 1];
        weight_count = node_counts[i] * node_counts[i + 1];
        layer->weights = (weight_t *)malloc(sizeof(weight_t) * weight_count);
        layer_randomise_weights(layer->weights, weight_count);
    }

    net->states = (data_t **)malloc(sizeof(data_t *) * (layer_count + 1));
    for (i = 1; i < layer_count; ++i) {
        net->states[i] = (data_t *)malloc(sizeof(data_t) * node_counts[i]);
    }

    net->errors = (data_t **)malloc(sizeof(data_t *) * (layer_count + 1));
    for (i = 0; i <= layer_count; ++i) {
        net->errors[i] = (data_t *)malloc(sizeof(data_t) * node_counts[i]);
    }

    return net;
}

void network_destroy(network_t *net)
{
    size_t i;

    for (i = 0; i <= net->layer_count; ++i) {
        free(net->errors[i]);
    }
    free(net->errors);

    for (i = 1; i < net->layer_count; ++i) {
        free(net->states[i]);
    }
    free(net->states);

    for (i = 0; i < net->layer_count; ++i) {
        free(net->layers[i].weights);
    }
    free(net->layers);

    free(net);
}

// Applies user-supplied input and output buffers for the NN to operate on.
void network_set_buffers(network_t *net, data_t *inputs, data_t *outputs)
{
    net->states[0] = inputs;
    net->states[net->layer_count] = outputs;
}

// Un-applies the buffers.
void network_reset_buffers(network_t *net)
{
    network_set_buffers(net, NULL, NULL);
}

void network_propagate_forward(network_t *net)
{
    size_t n;

    for (n = 0; n < net->layer_count; ++n) {
        layer_propagate_forward(&net->layers[n], net->states[n], net->states[n + 1]);
    }
}

data_t network_calculate_errors(network_t *net, const data_t *target)
{
    data_t total_error = 0.0;
    const layer_t *output_layer = &net->layers[net->layer_count - 1];
    data_t *outputs = net->states[net->layer_count];
    data_t *errors = net->errors[net->layer_count];
    data_t error;
    size_t n;

    for (n = 0; n < output_layer->output_count; ++n) {
        // NOTE: this was swapped to cancel the -1 for deltas
        // https://en.wikipedia.org/wiki/Backpropagation
        errors[n] = error = target[n] - outputs[n];
        total_error += fabs(error);
    }

    return total_error;
}

void network_propagate_errors(network_t *net)
{
    size_t n;

    for (n = net->layer_count - 1; n > 0; --n) {
        layer_propagate_errors(&net->layers[n], net->states[n], net->states[n + 1], net->errors[n + 1], net->errors[n]);
    }
    layer_propagate_errors(&net->layers[0], net->states[0], net->states[1], net->errors[1], net->errors[0]);
}

data_t network_train(network_t *net, const data_t *targets)
{
    data_t error;

    network_propagate_forward(net);
    error = network_calculate_errors(net, targets);
    network_propagate_errors(net);

    return error;
}

//
// Application code
//

typedef struct _network_fixture_t {
    network_t *net;
    size_t input_count;
    size_t output_count;
    data_t *inputs;
    data_t *outputs;
    data_t *targets;
} network_fixture_t;

network_fixture_t *fixture_create(network_t *net)
{
    network_fixture_t *fixture;

    fixture = (network_fixture_t *)malloc(sizeof(network_fixture_t));
    fixture->net = net;
    fixture->input_count = net->layers[0].input_count;
    fixture->output_count = net->layers[net->layer_count - 1].output_count;
    fixture->inputs = (data_t *)malloc(sizeof(data_t) * fixture->input_count);
    fixture->outputs = (data_t *)malloc(sizeof(data_t) * fixture->output_count);
    fixture->targets = (data_t *)malloc(sizeof(data_t) * fixture->output_count);
    network_set_buffers(net, fixture->inputs, fixture->outputs);
    return fixture;
}

void fixture_destroy(network_fixture_t *fixture)
{
    network_reset_buffers(fixture->net);
    free(fixture->targets);
    free(fixture->outputs);
    free(fixture->inputs);
    free(fixture);
}

typedef struct {
    const char *inputs;
    const char *targets;
} sample_t;

const sample_t samples[] = {
    // A
    {
        "0110" "1001" "1001" "1111" "1001" "1001",
        "10000",
    },
    // B
    {
        "1110" "1001" "1110" "1001" "1001" "1110",
        "01000",
    },
    // C
    {
        "0110" "1001" "1000" "1000" "1001" "0110",
        "00100",
    },
    // D
    {
        "1110" "1001" "1001" "1001" "1001" "1110",
        "00010",
    },
    // E
    {
        "1111" "1000" "1110" "1000" "1000" "1111",
        "00001",
    },
    { NULL, NULL },
};

void apply_sample(const sample_t *sample, const network_fixture_t *fixture)
{
    size_t n;

    for (n = 0; sample->inputs[n] != '\0'; ++n) {
        fixture->inputs[n] = sample->inputs[n] == '1' ? 1.0 : 0.0;
    }
    for (n = 0; sample->targets[n] != '\0'; ++n) {
        fixture->targets[n] = sample->targets[n] == '1' ? 1.0 : 0.0;
    }
}

#define PRINTOUT_TRAINING 1
#define PRINTOUT_TRAINING_CSV 1

void train(network_fixture_t *fixture, data_t max_total_error)
{
    data_t error, total_error;
    size_t epoch;
    size_t i;

#if PRINTOUT_TRAINING_CSV
    printf("EPOCH,ERROR\n");
#endif
    epoch = 0;
    do {
        ++epoch;
        total_error = 0.0;
        for (i = 0; i < 5; ++i) {
            apply_sample(&samples[i], fixture);
            error = network_train(fixture->net, fixture->targets);
            total_error += error;
        }
#if PRINTOUT_TRAINING_CSV
        printf("%d,%f\n", epoch, total_error);
#endif
    } while (total_error > max_total_error);
}