not_gate.py

xs = [1, 0]
ys = [0, 1]

w = 0.2
b = 0.5

def linear(x):
    return w * x + b
def relu(x):
    return max(x, 0)

def f(x):
    return relu(linear(x))

# l(x) = mx + b
# r(l) = max(l, 0)
# f(x) = r(l(x))

# derivative of the relu function:
# 1 if x > 0, else 0

# Backpropagation
# loss = (y - f(x))^2 

# dL / dw = dL / df * df / dl * dl / dw = 2(y - f(x)) df / dl * dl / dw 
# dL / dw = 2(y - f(x)) * r'(x) * x

# dL / db = 2(y - f(x)) * r'(x) * dl / db
# dL / db = 2(y - f(x)) * r'(x)

def drelu(x):
    return x >= 0

def dldw(x, y):
    return 2*(f(x) - y) * drelu(x) * x

def dldb(x, y):
    return 2*(f(x) - y) * drelu(x)
    
def loss(x, y):
    return (f(x) - y)**2
# m -= alpha * dldm(x, y)
# b -= alpha * dldb(x, y)

# Training
alpha = 0.01
for i in range(10000):
    x = xs[i%2]
    y = ys[i%2]
    print(loss(x, y))
    w -= alpha * dldw(x, y)
    b -= alpha * dldb(x, y)

Output exceeds the size limit. Open the full output data in a text editor
0.48999999999999994
0.26390823839999994
0.46588050895936
0.26710670198575487
0.44337182894256505
0.2699198192813085
0.42234967703297704
0.272364009951436
0.40270020014376623
0.27445575658234844
0.3843190464200694
0.27621149166208053
0.3671105228804591
0.2776475022596254
0.35098683105570705
0.27877985025287794
0.33586737318526155
0.279624306193047
0.32167812225526093
0.28019629510569793
0.3083510498141168
0.2805108527185532
0.2958236060897735
0.2805825907759615
0.2840382474630465
...
7.703719777548943e-30
6.749691120297282e-29
7.703719777548943e-30
6.749691120297282e-29 <- 6.74 * 10^-29