HandWriteNN2.cs

using System;

namespace HandWriteNN2
{
    class Program
    {
        static void Main(string[] args)
        {
            double lr = 0.1;

            double[] l2Input = { 1.0, 2.0, 3.0 };
            double[] teach = { 0.0, 1.0 };

            double[][] l1Conl2 = { new[] { 0.1, 0.1, 0.1 }, new[] { 0.2, 0.2, 0.2 } };
            double[][] l2Conl3 = { new[] { 1.0, 2.0 }, new[] { 3.0, 4.0 } };
            double[][] l3Conl4 = { new[] { 0.5, 0.3 }, new[] { 0.8, 0.6 } };

            double[] l2Bias = { 0, 0 };
            double[] l3Bias = { 0, 0 };
            double[] l4Bias = { 0, 0 };

            double[] l2Out = new double[2];
            l2Out[0] = Linear(l2Input, l1Conl2[0], l2Bias[0]);
            l2Out[1] = Linear(l2Input, l1Conl2[1], l2Bias[1]);

            Console.WriteLine("a_1\n[[" + l2Out[0] + " " + l2Out[1] + "]]");

            double[] l3Input = new double[2];
            l3Input[0] = Sigmoid(l2Out[0]);
            l3Input[1] = Sigmoid(l2Out[1]);

            Console.WriteLine("z\n[[" + l3Input[0] + " " + l3Input[1] + "]]");

            double[] l3Out = new double[2];
            l3Out[0] = Linear(l3Input, l2Conl3[0], l3Bias[0]);
            l3Out[1] = Linear(l3Input, l2Conl3[1], l3Bias[1]);

            Console.WriteLine("a_2\n[[" + l3Out[0] + " " + l3Out[1] + "]]");

            double[] l4Input = new double[2];
            l4Input[0] = Sigmoid(l3Out[0]);
            l4Input[1] = Sigmoid(l3Out[1]);

            Console.WriteLine("z2\n[[" + l4Input[0] + " " + l4Input[1] + "]]");

            double[] l4Out = new double[2];
            l4Out[0] = Linear(l4Input, l3Conl4[0], l4Bias[0]);
            l4Out[1] = Linear(l4Input, l3Conl4[1], l4Bias[1]);

            Console.WriteLine("y\n[[" + l4Out[0] + " " + l4Out[1] + "]]");

            //SquadError
            double[] l4Loss = new double[2];
            l4Loss[0] = SquadError(l4Out[0], teach[0]);
            l4Loss[1] = SquadError(l4Out[1], teach[1]);

            double loss = (l4Loss[0] * l4Loss[0] + l4Loss[1] * l4Loss[1]) / 2;
            Console.WriteLine("loss\n" + loss);

            double[][] dl3Conl4 = new double[2][] { new double[2], new double[2] };
            double[] dl4Bias = { 0, 0 };
            dl3Conl4[0] = DLinear(l4Loss[0], l4Input, ref dl4Bias[0]);
            dl3Conl4[1] = DLinear(l4Loss[1], l4Input, ref dl4Bias[1]);


            double[] dl3Sig = new double[2];
            dl3Sig[0] = DSigmoid(l4Input[0]);
            dl3Sig[1] = DSigmoid(l4Input[1]);

            double[] l3Loss = new double[2];
            l3Loss[0] = dl3Sig[0] * (l3Conl4[0][0] * l4Loss[0] + l3Conl4[1][0] * l4Loss[1]);
            l3Loss[1] = dl3Sig[1] * (l3Conl4[0][1] * l4Loss[0] + l3Conl4[1][1] * l4Loss[1]);

            double[][] dl2Conl3 = new double[2][] { new double[2], new double[2] };
            double[] dl3Bias = { 0, 0 };
            dl2Conl3[0] = DLinear(l3Loss[0], l3Input, ref dl3Bias[0]);
            dl2Conl3[1] = DLinear(l3Loss[1], l3Input, ref dl3Bias[1]);

            double[] dl2Sig = new double[2];
            dl2Sig[0] = DSigmoid(l3Input[0]);
            dl2Sig[1] = DSigmoid(l3Input[1]);

            double[] l2Loss = new double[2];
            l2Loss[0] = dl2Sig[0] * (l2Conl3[0][0] * l3Loss[0] + l2Conl3[1][0] * l3Loss[1]);
            l2Loss[1] = dl2Sig[1] * (l2Conl3[0][1] * l3Loss[0] + l2Conl3[1][1] * l3Loss[1]);

            double[][] dl1Conl2 = new double[2][] { new double[3], new double[3] };
            double[] dl2Bias = { 0, 0 };
            dl1Conl2[0] = DLinear(l2Loss[0], l2Input, ref dl2Bias[0]);
            dl1Conl2[1] = DLinear(l2Loss[1], l2Input, ref dl2Bias[1]);

            Console.WriteLine("W^1 grad\n[[" + dl1Conl2[0][0] + " " + dl1Conl2[0][1] + " " + dl1Conl2[0][2] + "]\n [" + dl1Conl2[1][0] + " " + dl1Conl2[1][1] + " " + dl1Conl2[1][2] + "]]");
            Console.WriteLine("W^2 grad\n[[" + dl2Conl3[0][0] + " " + dl2Conl3[0][1] + "]\n [" + dl2Conl3[1][0] + " " + dl2Conl3[1][1] + "]]");
            Console.WriteLine("W^3 grad\n[[" + dl3Conl4[0][0] + " " + dl3Conl4[0][1] + "]\n [" + dl3Conl4[1][0] + " " + dl3Conl4[1][1] + "]]");

            Console.WriteLine("before W^1\n[[" + l1Conl2[0][0] + " " + l1Conl2[0][1] + " " + l1Conl2[0][2] + "]\n [" + l1Conl2[1][0] + " " + l1Conl2[1][1] + " " + l1Conl2[1][2] + "]]");
            Console.WriteLine("before W^2\n[[" + l2Conl3[0][0] + " " + l2Conl3[0][1] + "]\n [" + l2Conl3[1][0] + " " + l2Conl3[1][1] + "]]");
            Console.WriteLine("before W^2\n[[" + l3Conl4[0][0] + " " + l3Conl4[0][1] + "]\n [" + l3Conl4[1][0] + " " + l3Conl4[1][1] + "]]");

            //Optimizer
            double[][] updatedl3Conl4 = new double[2][] { new double[2], new double[2] };
            updatedl3Conl4[0][0] = SGD(l3Conl4[0][0], lr, dl3Conl4[0][0]);
            updatedl3Conl4[0][1] = SGD(l3Conl4[0][1], lr, dl3Conl4[0][1]);
            updatedl3Conl4[1][0] = SGD(l3Conl4[1][0], lr, dl3Conl4[1][0]);
            updatedl3Conl4[1][1] = SGD(l3Conl4[1][1], lr, dl3Conl4[1][1]);

            double[] updatedl4Bias = new double[2];
            updatedl4Bias[0] = SGD(l4Bias[0], lr, dl4Bias[0]);
            updatedl4Bias[1] = SGD(l4Bias[1], lr, dl4Bias[1]);


            double[][] updatedl2Conl3 = new double[2][] { new double[2], new double[2] };
            updatedl2Conl3[0][0] = SGD(l2Conl3[0][0], lr, dl2Conl3[0][0]);
            updatedl2Conl3[0][1] = SGD(l2Conl3[0][1], lr, dl2Conl3[0][1]);
            updatedl2Conl3[1][0] = SGD(l2Conl3[1][0], lr, dl2Conl3[1][0]);
            updatedl2Conl3[1][1] = SGD(l2Conl3[1][1], lr, dl2Conl3[1][1]);

            double[] updatedl3Bias = new double[2];
            updatedl3Bias[0] = SGD(l3Bias[0], lr, dl3Bias[0]);
            updatedl3Bias[1] = SGD(l3Bias[1], lr, dl3Bias[1]);

            double[][] updatedl1Conl2 = new double[2][] { new double[3], new double[3] };
            updatedl1Conl2[0][0] = SGD(l1Conl2[0][0], lr, dl1Conl2[0][0]);
            updatedl1Conl2[0][1] = SGD(l1Conl2[0][1], lr, dl1Conl2[0][1]);
            updatedl1Conl2[0][2] = SGD(l1Conl2[0][2], lr, dl1Conl2[0][2]);
            updatedl1Conl2[1][0] = SGD(l1Conl2[1][0], lr, dl1Conl2[1][0]);
            updatedl1Conl2[1][1] = SGD(l1Conl2[1][1], lr, dl1Conl2[1][1]);
            updatedl1Conl2[1][2] = SGD(l1Conl2[1][2], lr, dl1Conl2[1][2]);

            double[] updatedl2Bias = new double[2];
            updatedl2Bias[0] = SGD(l2Bias[0], lr, dl2Bias[0]);
            updatedl2Bias[1] = SGD(l2Bias[1], lr, dl2Bias[1]);

            Console.WriteLine("after W^1\n[[" + updatedl1Conl2[0][0] + " " + updatedl1Conl2[0][1] + " " + updatedl1Conl2[0][2] + "]\n [" + updatedl1Conl2[1][0] + " " + updatedl1Conl2[1][1] + " " + updatedl1Conl2[1][2] + "]]");
            Console.WriteLine("after W^2\n[[" + updatedl2Conl3[0][0] + " " + updatedl2Conl3[0][1] + "]\n [" + updatedl2Conl3[1][0] + " " + updatedl2Conl3[1][1] + "]]");
            Console.WriteLine("after W^3\n[[" + updatedl3Conl4[0][0] + " " + updatedl3Conl4[0][1] + "]\n [" + updatedl3Conl4[1][0] + " " + updatedl3Conl4[1][1] + "]]");

            Console.Read();
        }

        static double Linear(double[] input, double[] w, double bias)
        {
            double result = 0;

            for (int i = 0; i < input.Length; i++)
            {
                result += input[i] * w[i];
            }

            return result + bias;
        }

        static double[] DLinear(double loss, double[] input, ref double bias)
        {
            double[] result = new double[input.Length];

            for (int i = 0; i < input.Length; i++)
            {
                result[i] = loss * input[i];
            }

            bias += loss;

            return result;
        }

        static double Sigmoid(double input)
        {
            return 1.0 / (1 + Math.Exp(-input));
        }

        static double DSigmoid(double input)
        {
            return (1.0 - input) * input;
        }

        //LossFunction
        static double SquadError(double input, double teach)
        {
            return input - teach;
        }

        //Optimaizer
        static double SGD(double param, double lr, double grad)
        {
            return param - lr * grad;
        }
    }
}