gradient descent example

gd.py

#!/usr/bin/python3

def gradient_descent(f, f_prime, x_0, alpha, num_iter=5):
    x_prev = x_0
    x = x_0
    for k in range(num_iter):
        x = x - alpha * f_prime(x)
        print("x=", x, "f(x)=", f(x), "diff=", x_prev - x)
        x_prev = x


if __name__ == "__main__":
    for alpha in [0, 1/5, 0.25, 1/2, 0.75, 1, 5]:
        print("CASE alpha=", alpha)
        gradient_descent(f=lambda x: x**2, f_prime=lambda x: 2*x,
                x_0=5, alpha=alpha, num_iter=15)
        print()

out.txt

CASE alpha= 0
x= 5 f(x)= 25 diff= 0
x= 5 f(x)= 25 diff= 0
x= 5 f(x)= 25 diff= 0
x= 5 f(x)= 25 diff= 0
x= 5 f(x)= 25 diff= 0
x= 5 f(x)= 25 diff= 0
x= 5 f(x)= 25 diff= 0
x= 5 f(x)= 25 diff= 0
x= 5 f(x)= 25 diff= 0
x= 5 f(x)= 25 diff= 0
x= 5 f(x)= 25 diff= 0
x= 5 f(x)= 25 diff= 0
x= 5 f(x)= 25 diff= 0
x= 5 f(x)= 25 diff= 0
x= 5 f(x)= 25 diff= 0

CASE alpha= 0.2
x= 3.0 f(x)= 9.0 diff= 2.0
x= 1.7999999999999998 f(x)= 3.2399999999999993 diff= 1.2000000000000002
x= 1.0799999999999998 f(x)= 1.1663999999999997 diff= 0.72
x= 0.6479999999999999 f(x)= 0.4199039999999999 diff= 0.43199999999999994
x= 0.3887999999999999 f(x)= 0.15116543999999993 diff= 0.2592
x= 0.23327999999999993 f(x)= 0.05441955839999997 diff= 0.15552
x= 0.13996799999999995 f(x)= 0.019591041023999986 diff= 0.09331199999999998
x= 0.08398079999999997 f(x)= 0.007052774768639994 diff= 0.05598719999999999
x= 0.05038847999999998 f(x)= 0.002538998916710398 diff= 0.03359231999999999
x= 0.030233087999999984 f(x)= 0.0009140396100157431 diff= 0.020155391999999994
x= 0.018139852799999988 f(x)= 0.0003290542596056674 diff= 0.012093235199999997
x= 0.010883911679999993 f(x)= 0.00011845953345804027 diff= 0.0072559411199999944
x= 0.006530347007999996 f(x)= 4.2645432044894506e-05 diff= 0.004353564671999997
x= 0.0039182082047999976 f(x)= 1.535235553616202e-05 diff= 0.0026121388031999987
x= 0.0023509249228799984 f(x)= 5.5268479930183265e-06 diff= 0.0015672832819199991

CASE alpha= 0.25
x= 2.5 f(x)= 6.25 diff= 2.5
x= 1.25 f(x)= 1.5625 diff= 1.25
x= 0.625 f(x)= 0.390625 diff= 0.625
x= 0.3125 f(x)= 0.09765625 diff= 0.3125
x= 0.15625 f(x)= 0.0244140625 diff= 0.15625
x= 0.078125 f(x)= 0.006103515625 diff= 0.078125
x= 0.0390625 f(x)= 0.00152587890625 diff= 0.0390625
x= 0.01953125 f(x)= 0.0003814697265625 diff= 0.01953125
x= 0.009765625 f(x)= 9.5367431640625e-05 diff= 0.009765625
x= 0.0048828125 f(x)= 2.384185791015625e-05 diff= 0.0048828125
x= 0.00244140625 f(x)= 5.9604644775390625e-06 diff= 0.00244140625
x= 0.001220703125 f(x)= 1.4901161193847656e-06 diff= 0.001220703125
x= 0.0006103515625 f(x)= 3.725290298461914e-07 diff= 0.0006103515625
x= 0.00030517578125 f(x)= 9.313225746154785e-08 diff= 0.00030517578125
x= 0.000152587890625 f(x)= 2.3283064365386963e-08 diff= 0.000152587890625

CASE alpha= 0.5
x= 0.0 f(x)= 0.0 diff= 5.0
x= 0.0 f(x)= 0.0 diff= 0.0
x= 0.0 f(x)= 0.0 diff= 0.0
x= 0.0 f(x)= 0.0 diff= 0.0
x= 0.0 f(x)= 0.0 diff= 0.0
x= 0.0 f(x)= 0.0 diff= 0.0
x= 0.0 f(x)= 0.0 diff= 0.0
x= 0.0 f(x)= 0.0 diff= 0.0
x= 0.0 f(x)= 0.0 diff= 0.0
x= 0.0 f(x)= 0.0 diff= 0.0
x= 0.0 f(x)= 0.0 diff= 0.0
x= 0.0 f(x)= 0.0 diff= 0.0
x= 0.0 f(x)= 0.0 diff= 0.0
x= 0.0 f(x)= 0.0 diff= 0.0
x= 0.0 f(x)= 0.0 diff= 0.0

CASE alpha= 0.75
x= -2.5 f(x)= 6.25 diff= 7.5
x= 1.25 f(x)= 1.5625 diff= -3.75
x= -0.625 f(x)= 0.390625 diff= 1.875
x= 0.3125 f(x)= 0.09765625 diff= -0.9375
x= -0.15625 f(x)= 0.0244140625 diff= 0.46875
x= 0.078125 f(x)= 0.006103515625 diff= -0.234375
x= -0.0390625 f(x)= 0.00152587890625 diff= 0.1171875
x= 0.01953125 f(x)= 0.0003814697265625 diff= -0.05859375
x= -0.009765625 f(x)= 9.5367431640625e-05 diff= 0.029296875
x= 0.0048828125 f(x)= 2.384185791015625e-05 diff= -0.0146484375
x= -0.00244140625 f(x)= 5.9604644775390625e-06 diff= 0.00732421875
x= 0.001220703125 f(x)= 1.4901161193847656e-06 diff= -0.003662109375
x= -0.0006103515625 f(x)= 3.725290298461914e-07 diff= 0.0018310546875
x= 0.00030517578125 f(x)= 9.313225746154785e-08 diff= -0.00091552734375
x= -0.000152587890625 f(x)= 2.3283064365386963e-08 diff= 0.000457763671875

CASE alpha= 1
x= -5 f(x)= 25 diff= 10
x= 5 f(x)= 25 diff= -10
x= -5 f(x)= 25 diff= 10
x= 5 f(x)= 25 diff= -10
x= -5 f(x)= 25 diff= 10
x= 5 f(x)= 25 diff= -10
x= -5 f(x)= 25 diff= 10
x= 5 f(x)= 25 diff= -10
x= -5 f(x)= 25 diff= 10
x= 5 f(x)= 25 diff= -10
x= -5 f(x)= 25 diff= 10
x= 5 f(x)= 25 diff= -10
x= -5 f(x)= 25 diff= 10
x= 5 f(x)= 25 diff= -10
x= -5 f(x)= 25 diff= 10

CASE alpha= 5
x= -45 f(x)= 2025 diff= 50
x= 405 f(x)= 164025 diff= -450
x= -3645 f(x)= 13286025 diff= 4050
x= 32805 f(x)= 1076168025 diff= -36450
x= -295245 f(x)= 87169610025 diff= 328050
x= 2657205 f(x)= 7060738412025 diff= -2952450
x= -23914845 f(x)= 571919811374025 diff= 26572050
x= 215233605 f(x)= 46325504721296025 diff= -239148450
x= -1937102445 f(x)= 3752365882424978025 diff= 2152336050
x= 17433922005 f(x)= 303941636476423220025 diff= -19371024450
x= -156905298045 f(x)= 24619272554590280822025 diff= 174339220050
x= 1412147682405 f(x)= 1994161076921812746584025 diff= -1569052980450
x= -12709329141645 f(x)= 161527047230666832473306025 diff= 14121476824050
x= 114383962274805 f(x)= 13083690825684013430337788025 diff= -127093291416450
x= -1029455660473245 f(x)= 1059778956880405087857360830025 diff= 1143839622748050