HandWriteNN.cs

using System;

namespace HandWriteNN
{
    class Program
    {
        static void Main(string[] args)
        {
            double lr = 0.1;

            double[] l2Input = { 1.0, 2.0, 3.0 };
            double[] teach = { 0.0, 1.0 };

            double[,] l1Conl2 = { { 0.1, 0.1, 0.1 }, { 0.2, 0.2, 0.2 } };
            double[,] l2Conl3 = { { 1, 2 }, { 3, 4 } };

            double[] l2Bias = { 0, 0 };
            double[] l3Bias = { 0, 0 };

            double[] l2Out = new double[2];
            l2Out[0] = l2Input[0] * l1Conl2[0, 0] + l2Input[1] * l1Conl2[0, 1] + l2Input[2] * l1Conl2[0, 2] + l2Bias[0];
            l2Out[1] = l2Input[0] * l1Conl2[1, 0] + l2Input[1] * l1Conl2[1, 1] + l2Input[2] * l1Conl2[1, 2] + l2Bias[1];
            Console.WriteLine("a_1\n[[" + l2Out[0] + " " + l2Out[1] + "]]");

            double[] l3Input = new double[2];
            l3Input[0] = Sigmoid(l2Out[0]);
            l3Input[1] = Sigmoid(l2Out[1]);
            Console.WriteLine("z\n[[" + l3Input[0] + " " + l3Input[1] + "]]");

            double[] l3Out = new double[2];
            l3Out[0] = l3Input[0] * l2Conl3[0, 0] + l3Input[1] * l2Conl3[0, 1] + l3Bias[0];
            l3Out[1] = l3Input[0] * l2Conl3[1, 0] + l3Input[1] * l2Conl3[1, 1] + l3Bias[1];
            Console.WriteLine("a_2\n[[" + l3Out[0] + " " + l3Out[1] + "]]");
            Console.WriteLine("y\n[[" + l3Out[0] + " " + l3Out[1] + "]]");

            //SquadError
            double[] l3Loss = new double[2];
            l3Loss[0] = SquadError(l3Out[0], teach[0]);
            l3Loss[1] = SquadError(l3Out[1], teach[1]);

            double loss = (l3Loss[0] * l3Loss[0] + l3Loss[1] * l3Loss[1]) / 2;
            Console.WriteLine("loss\n" + loss);

            double[,] dl2Conl3 = new double[2, 2];
            dl2Conl3[0, 0] = l3Loss[0] * l3Input[0];
            dl2Conl3[0, 1] = l3Loss[0] * l3Input[1];
            dl2Conl3[1, 0] = l3Loss[1] * l3Input[0];
            dl2Conl3[1, 1] = l3Loss[1] * l3Input[1];

            double[] dl3Bias = { 0, 0 };
            dl3Bias[0] += l3Loss[0];
            dl3Bias[1] += l3Loss[1];

            double[] dl2Sig = new double[2];
            dl2Sig[0] = DSigmoid(l3Input[0]);
            dl2Sig[1] = DSigmoid(l3Input[1]);

            double[,] dl1Conl2 = new double[2, 3];
            dl1Conl2[0, 0] = dl2Sig[0] * (l2Conl3[0, 0] * l3Loss[0] + l2Conl3[1, 0] * l3Loss[1]) * l2Input[0];
            dl1Conl2[0, 1] = dl2Sig[0] * (l2Conl3[0, 0] * l3Loss[0] + l2Conl3[1, 0] * l3Loss[1]) * l2Input[1];
            dl1Conl2[0, 2] = dl2Sig[0] * (l2Conl3[0, 0] * l3Loss[0] + l2Conl3[1, 0] * l3Loss[1]) * l2Input[2];
            dl1Conl2[1, 0] = dl2Sig[1] * (l2Conl3[0, 1] * l3Loss[0] + l2Conl3[1, 1] * l3Loss[1]) * l2Input[0];
            dl1Conl2[1, 1] = dl2Sig[1] * (l2Conl3[0, 1] * l3Loss[0] + l2Conl3[1, 1] * l3Loss[1]) * l2Input[1];
            dl1Conl2[1, 2] = dl2Sig[1] * (l2Conl3[0, 1] * l3Loss[0] + l2Conl3[1, 1] * l3Loss[1]) * l2Input[2];

            Console.WriteLine("W^1 grad\n[[" + dl1Conl2[0, 0] + " " + dl1Conl2[0, 1] + " " + dl1Conl2[0, 2] + "]\n [" + dl1Conl2[1, 0] + " " + dl1Conl2[1, 1] + " " + dl1Conl2[1, 2] + "]]");
            Console.WriteLine("W^2 grad\n[[" + dl2Conl3[0, 0] + " " + dl2Conl3[0, 1] + "]\n [" + dl2Conl3[1, 0] + " " + dl2Conl3[1, 1] + "]]");

            Console.WriteLine("before W^1\n[[" + l1Conl2[0, 0] + " " + l1Conl2[0, 1] + " " + l1Conl2[0, 2] + "]\n [" + l1Conl2[1, 0] + " " + l1Conl2[1, 1] + " " + l1Conl2[1, 2] + "]]");
            Console.WriteLine("before W^2\n[[" + l2Conl3[0, 0] + " " + l2Conl3[0, 1] + "]\n [" + l2Conl3[1, 0] + " " + l2Conl3[1, 1] + "]]");

            double[] dl2Bias = { 0, 0 };
            dl2Bias[0] += dl2Sig[0] * (l2Conl3[0, 0] * l3Loss[0] + l2Conl3[1, 0] * l3Loss[1]);
            dl2Bias[1] += dl2Sig[1] * (l2Conl3[0, 1] * l3Loss[0] + l2Conl3[1, 1] * l3Loss[1]);

            //Optimizer
            double[,] updatedl2Conl3 = new double[2, 2];
            updatedl2Conl3[0, 0] = SGD(l2Conl3[0, 0], lr, dl2Conl3[0, 0]);
            updatedl2Conl3[0, 1] = SGD(l2Conl3[0, 1], lr, dl2Conl3[0, 1]);
            updatedl2Conl3[1, 0] = SGD(l2Conl3[1, 0], lr, dl2Conl3[1, 0]);
            updatedl2Conl3[1, 1] = SGD(l2Conl3[1, 1], lr, dl2Conl3[1, 1]);

            double[] updatedl3Bias = new double[2];
            updatedl3Bias[0] = SGD(l3Bias[0], lr, dl3Bias[0]);
            updatedl3Bias[1] = SGD(l3Bias[1], lr, dl3Bias[1]);

            double[,] updatedl1Conl2 = new double[2, 3];
            updatedl1Conl2[0, 0] = SGD(l1Conl2[0, 0], lr, dl1Conl2[0, 0]);
            updatedl1Conl2[0, 1] = SGD(l1Conl2[0, 1], lr, dl1Conl2[0, 1]);
            updatedl1Conl2[0, 2] = SGD(l1Conl2[0, 2], lr, dl1Conl2[0, 2]);
            updatedl1Conl2[1, 0] = SGD(l1Conl2[1, 0], lr, dl1Conl2[1, 0]);
            updatedl1Conl2[1, 1] = SGD(l1Conl2[1, 1], lr, dl1Conl2[1, 1]);
            updatedl1Conl2[1, 2] = SGD(l1Conl2[1, 2], lr, dl1Conl2[1, 2]);

            double[] updatedl2Bias = new double[2];
            updatedl2Bias[0] = SGD(l2Bias[0], lr, dl2Bias[0]);
            updatedl2Bias[1] = SGD(l2Bias[1], lr, dl2Bias[1]);

            Console.WriteLine("after W^1\n[[" + updatedl1Conl2[0, 0] + " " + updatedl1Conl2[0, 1] + " " + updatedl1Conl2[0, 2] + "]\n [" + updatedl1Conl2[1, 0] + " " + updatedl1Conl2[1, 1] + " " + updatedl1Conl2[1, 2] + "]]");
            Console.WriteLine("after W^2\n[[" + updatedl2Conl3[0, 0] + " " + updatedl2Conl3[0, 1] + "]\n [" + updatedl2Conl3[1, 0] + " " + updatedl2Conl3[1, 1] + "]]");

            Console.Read();
        }

        static double Sigmoid(double input)
        {
            return 1.0 / (1 + Math.Exp(-input));
        }

        static double DSigmoid(double input)
        {
            return (1.0 - input) * input;
        }

        //LossFunction
        static double SquadError(double input, double teach)
        {
            return input - teach;
        }

        //Optimaizer
        static double SGD(double param, double lr, double grad)
        {
            return param - lr * grad;
        }
    }
}