back propagation expained

bp.py

import math

#back propagation expained

#http://cs231n.github.io/optimization-2/

def compute(w, x):
	# forward pass
	dot = w[0] * x[0] + w[1] * x[1] + w[2]
	# sigmoid
	f = 1.0 / (1 + math.exp(-dot))
	return f

def gradient(f, w, x):
	ddot = (1-f) * f #gradient on dot variable
	dw = [x[0] * ddot, x[1] * ddot, 1 * ddot] # chain rule [dw0, dw1, dw2]
	dx = [w[0] * ddot, w[1] * ddot] # chain rule [dx0, dx1]
	return dw, dx

w0 = [2,-3,-3]
x0 = [-1,-2]

# throw values to see gradient
# forward pass
f = compute(w0, x0)
print("f=%s" % f)

# backward pass check gradient
g_w0, g_x0 = gradient (f, w0, x0)
print("gw0=%s" % g_w0)
print("gx0=%s" % g_x0)

def evaluate(w_chg, x_chg):
	w1 = [w0[0] + w_chg[0], w0[1]+ w_chg[1], w0[2]+ w_chg[2]]
	x1 = [x0[0]+ x_chg[0], x0[1]+ x_chg[1]]
	
	expected_f = f + w_chg[0] * g_w0[0] + w_chg[1] * g_w0[1] + w_chg[2] * g_w0[2] + x_chg[0] * g_x0[0] + x_chg[1] * g_x0[1]

	print("expectedf=%s" % expected_f)
	actual_f = compute(w1,x1)
	print("actual_f=%s" % actual_f)
	diff = actual_f - expected_f
	#print("diff=%s" % diff)
	return diff

# should be approximatly zero when those change is very small
# according to the gradient at w0,x0
#                 w_change     x_change
print(evaluate([0.2,0.1,0.1],[0.1,0.2]))