A small example of computing the gradient of a function by backpropagation.

backpropagationExample.cpp

#include <iostream>
#include <cmath>

using std::cout;
using std::endl;

struct vec2
{
	double x;
	double y;
};

double sigmoid(double x)
{
	return 1./(1. + exp(-x));
}

double f(vec2 p)
{
	return (sigmoid(p.y) + p.x)/(sigmoid(p.x) + pow(p.x+p.y, 2));
}

static const double EPS = 1e-6;
vec2 numericalGrad(vec2 p)
{
	vec2 ret;

	vec2 orig_p = p;
	vec2 p2 = p;

	p.x += EPS;
	p2.x -= EPS;
	ret.x = (f(p) - f(p2))/EPS/2.;

	p = orig_p;
	p2 = orig_p;

	p.y += EPS;
	p2.y -= EPS;
	ret.y = (f(p) - f(p2))/EPS/2.;

	return ret;
}

vec2 backPropGrad_f(vec2 p)
{
	double f_direct = f(p);

	double a2 = p.x + p.y;
	double d = a2*a2;
	double sigx = sigmoid(p.x);
	double a0 = sigx + d;
	double c = 1./a0;
	double sigy = sigmoid(p.y);
	double a1 = sigy + p.x;
	double f_forward = a1 * c;

	cout << "forward " << f_forward << endl;
	cout << "direct " << f_direct << endl;

	double dc = a1;
	double da0 = -1./(a0*a0) * dc;
	double dsigx = 1. * da0;
	double dx = (1.-sigx)*sigx * dsigx;

	double dd = 1. * da0;
	double da2 = 2*a2 * dd;
	dx += 1. * da2;
	double dy = 1. * da2;

	double da1 = c;
	dx += 1. * da1;
	double dsigy = 1. * da1;
	dy += (1.-sigy)*sigy * dsigy;

	p.x = dx;
	p.y = dy;
	return p;
}

int main()
{
	vec2 p = {12., 4.};
	vec2 bp = backPropGrad_f(p);
	cout << "backpropagation {" << bp.x << ", " << bp.y << "}" << endl;
	p = numericalGrad(p);
	cout << "numericalGrad {" << p.x << ", " << p.y << "}" << endl;

	p = {0., 4.};
	bp = backPropGrad_f(p);
	cout << "backpropagation {" << bp.x << ", " << bp.y << "}" << endl;
	p = numericalGrad(p);
	cout << "numericalGrad {" << p.x << ", " << p.y << "}" << endl;

	p = {10., -4.};
	bp = backPropGrad_f(p);
	cout << "backpropagation {" << bp.x << ", " << bp.y << "}" << endl;
	p = numericalGrad(p);
	cout << "numericalGrad {" << p.x << ", " << p.y << "}" << endl;
}