Failed attempts to do machine learning using gennan

example_audio.c

#include <stdio.h>
#include <stdlib.h>
#include <time.h>

#include "genann.h"

#include "stb_vorbis.c"

#define AUDIO_WINDOW 128
#define AUDIO_OUTPUT (AUDIO_WINDOW * 2)

int main(int argc, char *argv[]) {
  if (argc != 2) {
    printf("usage: %s <audio file>\n", argv[0]);
    return 1;
  }

  int error;
  stb_vorbis_alloc *sound_alloc = calloc(1, sizeof(stb_vorbis_alloc));
  stb_vorbis *vorbis = stb_vorbis_open_filename(argv[1], &error, sound_alloc);
  stb_vorbis_info info = stb_vorbis_get_info(vorbis);
  size_t size = info.channels * stb_vorbis_stream_length_in_samples(vorbis) *
                sizeof(float);

  // Audio stb_vorbis
  float *samples = malloc(size);

  // Read audio file
  stb_vorbis_get_samples_float_interleaved(
      vorbis, info.channels, (float *)samples, vorbis->total_samples);
  // Close
  stb_vorbis_close(vorbis);

  // Print number of samples
  printf("Number of samples: %d\n", vorbis->total_samples);
  // Print window size
  printf("Window size: %d\n", AUDIO_WINDOW);
  // Print output size
  printf("Output size: %d\n", AUDIO_OUTPUT);
  size_t number_of_windows = (vorbis->total_samples / AUDIO_OUTPUT) - 1;
  printf("Number of windows: %ld\n", number_of_windows);

  /* This will make the neural network initialize differently each run. */
  /* If you don't get a good result, try again for a different result. */
  srand(time(0));

  /* Input and expected out data for the XOR function. */
  double input[AUDIO_WINDOW];
  double output[AUDIO_OUTPUT];

  /* New network */
  genann *ann = genann_init(AUDIO_WINDOW, 4, AUDIO_OUTPUT * 4, AUDIO_OUTPUT);

  for (size_t i = 1000; i < 1500; i++) {
    // Copy input, but skip 1/2 of the samples
    for (size_t j = 0; j < AUDIO_WINDOW; j++) {
      size_t index = (i * AUDIO_OUTPUT) + (j * 2);
      input[j] = (double)samples[index];
    }
    // Copy output as is
    for (size_t j = 0; j < AUDIO_OUTPUT; j++) {
      output[j] = (double)samples[(i * AUDIO_OUTPUT) + j];
    }

    // Verify input and output
    assert(input[0] == output[0]);
    assert(input[1] == output[2]);

    genann_train(ann, input, output, 1);
    if (i % 100 == 0)
        printf("Trained %ld\n", i);
  }

  printf("Trained!\n");
  // Save network
  printf("Saving network...\n");
  // genann_write(ann, FILE *out)
  FILE *fp = fopen("network.bin", "wb");
  genann_write(ann, fp);
  fclose(fp);
  printf("Saved!\n");

  /* Run the network and see what it predicts. */
  // Take a random window
  size_t random_window = rand() % number_of_windows;
  printf("Random window: %ld\n", random_window);
  // Copy input skip 1/2
  printf("Getting input...\n");
  for (size_t j = 0; j < AUDIO_WINDOW; j++) {
    size_t index = (random_window * AUDIO_OUTPUT) + (j * 2);
    input[j] = (double)samples[index];
  }
  printf("Getting output...\n");
  // Real output
  for (size_t j = 0; j < AUDIO_WINDOW; j++) {
    output[j] = (double)samples[(random_window * AUDIO_OUTPUT) + j];
  }

  size_t nn_error = 0;
  double margin = 0.01;

  // Output predicted
  double *predicted_output = genann_run(ann, input);

  // Replace predicted output with real output 1/2 samples
  for (size_t j = 0; j < AUDIO_OUTPUT; j += 2) {
    predicted_output[j] = output[j];
  }

  // Calculate error
  for (size_t j = 0; j < AUDIO_OUTPUT; j++) {
    if (predicted_output[j] > (output[j] + margin) ||
        predicted_output[j] < (output[j] - margin)) {
      nn_error += 1;
    }
  }
  error /= AUDIO_OUTPUT;
  printf("Errors: %ld\n", nn_error);
  printf("Error p: %f\n", (double)nn_error / (double)AUDIO_OUTPUT);
  printf("Error on predicted: %f\n",
         ((double)nn_error / (double)AUDIO_OUTPUT) * 2.0);

  genann_free(ann);
  free(samples);
  return 0;
}

examplenoise.c

#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <assert.h>
#include <math.h>
#include "genann.h"

struct matrix {
    int rows;
    int cols;
    float *data;
};

struct delta_results {
    size_t errors;
    float average_delta;
    float max_delta;
    float min_delta;
};

struct matrix *matrix_create(int rows, int cols)
{
    struct matrix *m = malloc(sizeof(struct matrix));
    m->rows = rows;
    m->cols = cols;
    m->data = calloc(rows * cols, sizeof(float));
    assert(m->data != NULL);
    return m;
}

struct matrix *matrix_create_raw(int rows, int cols, float *data) {
    struct matrix *m = malloc(sizeof(struct matrix));
    m->rows = rows;
    m->cols = cols;
    m->data = malloc(sizeof(float) * rows * cols);
    assert(m->data != NULL);
    // Copy data
    for (int i = 0; i < rows * cols; i++) {
        m->data[i] = data[i];
    }
    return m;
}

void matrix_free(struct matrix *m)
{
    free(m->data);
    free(m);
}

void matrix_print(struct matrix *m)
{
    int i, j;
    for (i = 0; i < m->rows; ++i) {
        for (j = 0; j < m->cols; ++j) {
            printf("%1.3f ", m->data[i * m->cols + j]);
        }
        printf("\n");
    }
}

float matrix_get(struct matrix *m, int row, int col)
{
    assert(row >= 0 && row < m->rows);
    assert(col >= 0 && col < m->cols);
    assert(m != NULL);
    assert(m->data != NULL);
    return m->data[row * m->cols + col];
}

void matrix_set(struct matrix *m, int row, int col, float val)
{
    assert(row >= 0 && row < m->rows);
    assert(col >= 0 && col < m->cols);

    m->data[row * m->cols + col] = val;
}

// Generate a 2d -> 1d convolution from the matrix
float *matrix_convolution(struct matrix *mat, size_t kernel_size, size_t *output_size) {
    size_t size = (mat->rows - kernel_size + 1) * (mat->cols - kernel_size + 1);
    printf("Size: %zu\n", size);
    *output_size = size;
    float *output = malloc(sizeof(float) * size);

    assert(output != NULL);

    // For each row
    for (int j = 0; j < mat->rows - kernel_size + 1; j++) {
        // For each column
        for (int k = 0; k < mat->cols - kernel_size + 1; k++) {
            // For each kernel row
            for (int l = 0; l < kernel_size; l++) {
                // For each kernel column
                for (int m = 0; m < kernel_size; m++) {
                    output[j * (mat->cols - kernel_size + 1) + k] += matrix_get(mat, j + l, k + m);
                }
            }
        }
    }

    return output;
}

#if 0
void generate_noise_matrixes(struct matrix *input, struct matrix *output, size_t size)
{
    // Fill matrix with normalized number
    for (int j = 0; j < size; j++) {
        for (int k = 0; k < size; k++) {

            // Choose a random number between 0 and 255
            int random_number = rand() % 256;

            // Normalize between 0 and 1
            float normalized_number = random_number / 255.0;

            // Add noise between -0.1 and 0.1
            float noise = (rand() % 200 - 100) / 1000.0;
            float final_value = normalized_number + noise;

            matrix_set(output, j, k, normalized_number);
            matrix_set(input, j, k, final_value < 0 ? 0 : final_value);
        }
    }
}
#endif

void generate_noise_matrixes(struct matrix *input, struct matrix *output, size_t size)
{
    int random_number = rand() % 256;
    float normalized_number = random_number / 255.0;

    // Fill matrix with normalized number
    for (int j = 0; j < size; j++) {
        for (int k = 0; k < size; k++) {

            size_t random_hole = rand() % size;

            matrix_set(output, j, k, normalized_number);
            matrix_set(input, j, k, random_hole <= (size / 4) ? -1.0 : normalized_number);
        }
    }
}

struct delta_results compute_delta(struct matrix *input, struct matrix *output, size_t size, float delta_max)
{
    struct delta_results results = {0, 0, 0, 1.0};

    // Compute delta
    for (int j = 0; j < size; j++) {
        for (int k = 0; k < size; k++) {

            // Compute delta
            float delta = fabs(matrix_get(output, j, k) - matrix_get(input, j, k));

            // Update results
            results.average_delta += delta;
            if (delta > results.max_delta) {
                results.max_delta = delta;
            }
            if (delta < results.min_delta) {
                results.min_delta = delta;
            }
            if (delta > delta_max) {
                results.errors++;
            }
        }
    }

    results.average_delta /= size * size;

    return results;
}

// Analyse the matrix
float bullshit_algorithm(struct matrix *input, size_t size)
{
    // Compute average value
    float average = 0;
    for (int j = 0; j < size; j++) {
        for (int k = 0; k < size; k++) {
            average += matrix_get(input, j, k);
        }
    }

    // Get the minimum and maximum values
    float min = matrix_get(input, 0, 0);
    float max = matrix_get(input, 0, 0);

    for (int j = 0; j < size; j++) {
        for (int k = 0; k < size; k++) {
            float value = matrix_get(input, j, k);
            if (value < min) {
                min = value;
            }
            if (value > max) {
                max = value;
            }
        }
    }

    // Use the average, min and max to compute a score
    average /= size * size;
    float score = (average - min) / (max - min);

    return score;
}

int main(int argc, char *argv[])
{
    const size_t max_iter = 1000;
    const size_t matrix_number = 300;
    const size_t validation_number = matrix_number / 4;
    const size_t matrix_size = 16;
    const float learning_rate = 2.1;
    // Tested matrixes
    const size_t required_training_matrixes = 10;
    // Should succeed treshold
    const size_t required_validation_matrixes = 8;
    float delta_max = 0.01;
    genann *ann = genann_init(matrix_size * matrix_size, 1, 386, matrix_size * matrix_size);

    /* This will make the neural network initialize differently each run. */
    /* If you don't get a good result, try again for a different result. */
    srand(time(0));

    // 8x8 matrix -> 64 inputs
    // Generate 8x8 matrix of one random number

    struct matrix **input = malloc (sizeof(struct matrix*) * matrix_number);
    struct matrix **output = malloc (sizeof(struct matrix*) * matrix_number);

    for (int i = 0; i < matrix_number; i++) {
        // Create matrixes
        input[i] = matrix_create(matrix_size, matrix_size);
        output[i] = matrix_create(matrix_size, matrix_size);

        generate_noise_matrixes(input[i], output[i], matrix_size);

    }

    // Take a random matrix and print
    size_t rand_matrix = rand() % matrix_number;
    printf("Matrix size: %ld\n", matrix_size);
#if 0
    printf("Random matrix: %ld\n", rand_matrix);
    printf("Input:\n");
    matrix_print(input[rand_matrix]);
    printf("Output:\n");
    matrix_print(output[rand_matrix]);
#endif
    size_t epoch = 0;
    /* Train on the four labeled data points many times. */
    for (epoch = 0; epoch < max_iter; epoch++) {
        for (size_t current_mat = 0; current_mat < matrix_number; current_mat++) {
            // Copy input and output
            float input_data[matrix_size * matrix_size];
            float output_data[matrix_size * matrix_size];

            // Copy data
            for (size_t j = 0; j < matrix_size; j++) {
                for (size_t k = 0; k < matrix_size; k++) {
                    float input_value = matrix_get(input[current_mat], j, k);
                    float output_value = matrix_get(output[current_mat], j, k);
                    input_data[j * matrix_size + k] = input_value;
                    output_data[j * matrix_size + k] = output_value;
                }
            }

            genann_train(ann, input_data, output_data, learning_rate);
        }


        size_t tested_matrix = 0;
        size_t last_train_errors = 0;

        for (size_t mat = 0; mat < required_training_matrixes; mat++) {
            // Take the current matrix and compute the delta
            rand_matrix = rand() % matrix_number;
            struct matrix *ann_matrix = matrix_create_raw(matrix_size, matrix_size, genann_run(ann, input[rand_matrix]->data));
            struct delta_results dt_results = compute_delta(output[rand_matrix], ann_matrix, matrix_size, delta_max);
            last_train_errors += dt_results.errors;
            if (dt_results.average_delta <= delta_max) {
                tested_matrix++;
            }
        }

        if (tested_matrix >= required_training_matrixes) {
            printf("Training succeeded at epoch %ld\n", epoch);
           break;
        }

        printf("Epoch %ld\n", epoch);
        printf("Errors currently (rnd matrix): %ld\n", last_train_errors);

     }

    /* Run the network and see what it predicts. */
    // Generate 8x8 matrix of one random number
    struct matrix *test_input = matrix_create(matrix_size, matrix_size);
    struct matrix *test_output = matrix_create(matrix_size, matrix_size);
    // Generate 8x8 matrix of one random number
    generate_noise_matrixes(test_input, test_output, matrix_size);

    // Feed to network
    float *ann_output = genann_run(ann, test_input->data);
    struct matrix *ann_matrix = matrix_create_raw(matrix_size, matrix_size, ann_output);

    // Compute delta
    struct delta_results dt_results = compute_delta(test_output, ann_matrix, matrix_size, delta_max);

    // Print delta

    printf("Input:\n");
    matrix_print(test_input);

    printf("Output:\n");
    matrix_print(test_output);

    printf("Predicted:\n");
    matrix_print(ann_matrix);

    printf("Errors: %ld on %ld\n", dt_results.errors, matrix_size * matrix_size);
    printf("Max delta: %f\n", dt_results.max_delta);
    printf("Min delta: %f\n", dt_results.min_delta);
    printf("Precision %f\n", 1 - (float)dt_results.errors / (matrix_size * matrix_size));
    printf("Average delta: %f\n", dt_results.average_delta);
    printf("Required epoch: %ld\n", epoch);
    printf("Required delta: %f\n", delta_max);

    // Free memory
    for (int i = 0; i < matrix_number; i++) {
        matrix_free(input[i]);
        matrix_free(output[i]);
    }
    free(input);
    free(output);

    printf("Saving network to file...\n");
    FILE *fp = fopen("network.bin", "wb");
    genann_write(ann, fp);
    fclose(fp);

    genann_free(ann);
    return 0;
}