MNIST Classifier implementation

Mnist.cc

#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <fstream>
#include <functional>
#include <random>
#include <stdexcept>
#include <vector>

float const eta = 0.01;

size_t const training_num = 20;

float FrontRelu(float f)
{
    if (f > 0)
        return f;
    else
        return 0.0f;
}

float BackRelu(float f)
{
    if (f > 0)
        return 1.0f;
    else
        return 0.0f;
}

template <typename It>
It FindMax(It begin, It end)
{
    It max = begin;
    for (++begin; begin != end; ++begin)
    {
        if (*max < *begin)
            max = begin;
    }
    return max;
}

struct Layer
{
    std::vector<float> data;

    size_t size;

    Layer(size_t layer_size) : data(layer_size), size(layer_size) {}

    float &      At(size_t idx) { return data.at(idx); }
    float const &At(size_t idx) const { return data.at(idx); }
};

struct Affine
{
    std::vector<float> weight;
    std::vector<float> bias;

    size_t inputSize, outputSize;

    Affine(size_t input_size, size_t output_size)
        : weight(input_size * output_size),
          bias(output_size),
          inputSize(input_size),
          outputSize(output_size)
    {
        std::normal_distribution<float> dis;
        std::default_random_engine      eng;

        for (float &f : weight) f = dis(eng);
        for (float &f : bias) f = dis(eng);
    }

    void Front(Layer const &in, Layer &out)
    {
        if (inputSize != in.size)
            throw std::exception();

        if (outputSize != out.size)
            throw std::exception();

        for (size_t i = 0; i < outputSize; ++i)
        {
            float e = 0.0f;

            for (size_t j = 0; j < inputSize; ++j)
                e += weight.at(i * inputSize + j) * in.At(j);
            e += bias.at(i);

            out.At(i) = FrontRelu(e);
        }
    }

    void Back(Layer const &in, Layer const &back, Layer &front)
    {
        if (inputSize != in.size)
            throw std::exception();

        if (inputSize != front.size)
            throw std::exception();

        if (outputSize != back.size)
            throw std::exception();

        for (size_t i = 0; i < inputSize; ++i)
        {
            if (BackRelu(in.At(i)) == 0.0f)
                front.At(i) = 0.0f;
            else
            {
                float e = 0.0f;

                for (size_t j = 0; j < outputSize; ++j)
                    e += weight.at(i + j * inputSize) * back.At(j);

                front.At(i) = e;
            }
        }

        for (size_t i = 0; i < outputSize; ++i)
        {
            for (size_t j = 0; j < inputSize; ++j)
                weight.at(i * inputSize + j) -= eta * in.At(j) * back.At(i);

            bias.at(i) -= eta * back.At(i);
        }
    }
};

struct SoftmaxWithLoss
{
    void Front(Layer const &in, Layer const &label, Layer &out, float &error)
    {
        if (in.size != label.size)
            throw std::exception();

        if (in.size != out.size)
            throw std::exception();

        float max = *FindMax(in.data.begin(), in.data.end());
        float sum = 0.0f;
        for (float f : in.data) sum += exp(f - max);

        error = 0.0f;
        for (size_t i = 0; i < out.size; ++i)
        {
            out.At(i) = exp(in.At(i) - max) / sum;
            error -= 1.f * label.At(i) * log(out.At(i));
        }
    }

    void Back(Layer const &out, Layer const &label, Layer &front)
    {
        if (out.size != label.size)
            throw std::exception();

        if (out.size != front.size)
            throw std::exception();

        for (size_t i = 0; i < out.size; ++i)
        { front.At(i) = out.At(i) - label.At(i); }
    }
};

struct Mnist : public Layer
{
    enum Label
    {
        Zero,
        One,
        Two,
        Three,
        Four,
        Five,
        Six,
        Seven,
        Eight,
        Nine
    } label;

    Mnist() : Layer(0), label(Zero) {}

    Mnist(unsigned char *ptr, Label label) : Layer(28 * 28), label(label)
    {
        for (size_t i = 0; i < 28 * 28; ++i) data[i] = ptr[i] / 255.0f;
    }

    void SetLabel(Layer &l) const
    {
        for (float &f : l.data) f = 0.0f;

        l.At(label) = 1.0f;
    }
};

void RevertEndian(int &endian)
{
    union
    {
        char b[4];
        int  i;
    } u, v;
    u.i    = endian;
    v.b[0] = u.b[3];
    v.b[1] = u.b[2];
    v.b[2] = u.b[1];
    v.b[3] = u.b[0];
    endian = v.i;
}

void ReadMnist(std::string const &image_path, std::string const &label_path,
               std::vector<Mnist> &train_list)
{
    std::ifstream image_read(image_path, std::ifstream::binary),
        label_read(label_path, std::ifstream::binary);

    if (!image_read || !label_read)
        throw std::exception();

    int image_magic, label_magic;
    image_read.read(reinterpret_cast<char *>(&image_magic), 4);
    label_read.read(reinterpret_cast<char *>(&label_magic), 4);
    if (image_magic != 0x03080000)
        throw std::exception();
    if (label_magic != 0x01080000)
        throw std::exception();

    int image_len, label_len;
    image_read.read(reinterpret_cast<char *>(&image_len), 4);
    label_read.read(reinterpret_cast<char *>(&label_len), 4);
    RevertEndian(image_len);
    RevertEndian(label_len);
    if (image_len != label_len)
        throw std::exception();

    int image_rows, image_columns;
    image_read.read(reinterpret_cast<char *>(&image_rows), 4);
    image_read.read(reinterpret_cast<char *>(&image_columns), 4);
    RevertEndian(image_rows);
    RevertEndian(image_columns);
    if (image_rows != 28 || image_columns != 28)
        throw std::exception();

    train_list = std::vector<Mnist>(image_len);
    std::vector<char> images(28 * 28 * image_len);
    std::vector<char> labels(label_len);
    image_read.read(images.data(), 28 * 28 * image_len);
    label_read.read(labels.data(), label_len);

    image_read.close();
    label_read.close();

    for (int i = 0; i < image_len; ++i)
    {
        train_list.at(i) = Mnist(
            reinterpret_cast<unsigned char *>(images.data()) + 28 * 28 * i,
            static_cast<Mnist::Label>(labels.at(i)));
    }
}

int main()
{
    printf("Reading data...\n");

    std::vector<Mnist> train_list, test_list;

    ReadMnist("train-images.idx3-ubyte", "train-labels.idx1-ubyte", train_list);
    ReadMnist("t10k-images.idx3-ubyte", "t10k-labels.idx1-ubyte", test_list);

    Layer           l1(50), l2(100), y(10), label(10), out(10);
    Layer           p(28 * 28), b1(50), b2(100), q(10);
    Affine          a1(28 * 28, 50), a2(50, 100), a3(100, 10);
    SoftmaxWithLoss soft;
    float           error;

    for (size_t _ = 1; _ <= training_num; ++_)
    {
        // Training
        for (size_t i = 0, l = train_list.size(); i < l; ++i)
        {
            auto const &mnist = train_list.at(i);

            mnist.SetLabel(label);

            a1.Front(mnist, l1);
            a2.Front(l1, l2);
            a3.Front(l2, y);
            soft.Front(y, label, out, error);

            soft.Back(out, label, q);
            a3.Back(l2, q, b2);
            a2.Back(l1, b2, b1);
            a1.Back(mnist, b1, p);

            printf("Training(%zd)... Error: %f\t%zd of %zd\n", _, error, i, l);
        }
    }

    // Evaluation
    size_t correct = 0;
    for (size_t i = 0, l = train_list.size(); i < l; ++i)
    {
        auto const &mnist = train_list.at(i);

        mnist.SetLabel(label);

        a1.Front(mnist, l1);
        a2.Front(l1, l2);
        a3.Front(l2, y);
        soft.Front(y, label, out, error);

        Mnist::Label idx = static_cast<Mnist::Label>(std::distance(
            out.data.begin(), FindMax(out.data.begin(), out.data.end())));
        if (idx == mnist.label)
            ++correct;

        printf("Evaluating... Error: %f\tAccuracy: %f\t%zd of %zd\n", error,
               (float)correct / i, i, l);
    }

    return 0;
}