Skip to content

Instantly share code, notes, and snippets.

@liginity
Last active July 3, 2023 01:58
Show Gist options
  • Save liginity/c37a1c17ff617d65d9ad16afacfe9714 to your computer and use it in GitHub Desktop.
Save liginity/c37a1c17ff617d65d9ad16afacfe9714 to your computer and use it in GitHub Desktop.
use pytorch functionality to realize gradient descent on linear mapping (or linear regression).
"""use pytorch in this example.
input: dimension = (16,)
output: dimension = (4,)
matrix of linear mapping: dimension = (16, 4)
"""
import torch
from torch import nn
# variables for dimensions
d1 = 64
d2 = 32
d3 = 16
d4 = 4
d5 = 1
# number of steps for gradient descent.
n_steps = 5
learning_rate = 0.0005
# the linear mapping matrix
W2 = nn.Parameter(torch.zeros((d3, d4), dtype=torch.float64, requires_grad=True))
# optimizer and loss function.
# stochastic gradient descent optimizer
optimizer2 = torch.optim.SGD([W2], lr=learning_rate)
# print(optimizer2)
# mean squared error (squared L2 norm)
loss_fn2 = torch.nn.MSELoss()
# input is x2, output is y2.
# x2 is 1D array. length of x2 = d3.
x2 = torch.tensor([i for i in range(-d3, d3, 2)], dtype=torch.float64)
print(f"input:\n x2 = {x2}")
print(f" x2.shape = {x2.shape}")
print()
nums = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53]
# y2 is 1D array. length of y2 = d4.
y2 = torch.tensor(nums[:d4], dtype=torch.float64)
print(f"output:\n y2 = {y2}")
print(f" y2.shape = {y2.shape}")
print()
# before gradient descent
print("before gradient descent")
print("W2 (W2 is the linear mapping matrix) =")
print(W2)
print(f"W2.shape = {W2.shape}")
print()
# do gradient descent for n steps
print("do gradient descent")
for step in range(n_steps):
print("-" * 80)
print(f"step {step:>2}")
# zero the stored gradient
optimizer2.zero_grad()
# make a prediction
## use tensordot to calculate the prediction
# y2_prediction_stepi = torch.tensordot(x2, W2, dims=[[-1], [0]])
## use matrix multiplication to calculate the prediction
## dimension: (16) x (16, 4) -> (4)
y2_prediction_stepi = x2 @ W2
# calculate the loss between prediction and target
loss2_stepi = loss_fn2(y2_prediction_stepi, y2)
print(f"loss = {loss2_stepi.item()}")
# backward propagation
loss2_stepi.backward()
# this adjusts the parameters in the matrix
optimizer2.step()
print("W2 (W2 is the linear mapping matrix) =")
print(W2)
@liginity
Copy link
Author

liginity commented Jul 3, 2023

Execute the code and show output.

$ python gradient-descent-on-linear-mapping.py
input:
  x2 = tensor([-16., -14., -12., -10.,  -8.,  -6.,  -4.,  -2.,   0.,   2.,   4.,   6.,
          8.,  10.,  12.,  14.], dtype=torch.float64)
  x2.shape = torch.Size([16])

output:
  y2 = tensor([2., 3., 5., 7.], dtype=torch.float64)
  y2.shape = torch.Size([4])

before gradient descent
W2 (W2 is the linear mapping matrix) =
Parameter containing:
tensor([[0., 0., 0., 0.],
        [0., 0., 0., 0.],
        [0., 0., 0., 0.],
        [0., 0., 0., 0.],
        [0., 0., 0., 0.],
        [0., 0., 0., 0.],
        [0., 0., 0., 0.],
        [0., 0., 0., 0.],
        [0., 0., 0., 0.],
        [0., 0., 0., 0.],
        [0., 0., 0., 0.],
        [0., 0., 0., 0.],
        [0., 0., 0., 0.],
        [0., 0., 0., 0.],
        [0., 0., 0., 0.],
        [0., 0., 0., 0.]], dtype=torch.float64, requires_grad=True)
W2.shape = torch.Size([16, 4])

do gradient descent
--------------------------------------------------------------------------------
step  0
loss = 21.75
W2 (W2 is the linear mapping matrix) =
Parameter containing:
tensor([[-0.0080, -0.0120, -0.0200, -0.0280],
        [-0.0070, -0.0105, -0.0175, -0.0245],
        [-0.0060, -0.0090, -0.0150, -0.0210],
        [-0.0050, -0.0075, -0.0125, -0.0175],
        [-0.0040, -0.0060, -0.0100, -0.0140],
        [-0.0030, -0.0045, -0.0075, -0.0105],
        [-0.0020, -0.0030, -0.0050, -0.0070],
        [-0.0010, -0.0015, -0.0025, -0.0035],
        [ 0.0000,  0.0000,  0.0000,  0.0000],
        [ 0.0010,  0.0015,  0.0025,  0.0035],
        [ 0.0020,  0.0030,  0.0050,  0.0070],
        [ 0.0030,  0.0045,  0.0075,  0.0105],
        [ 0.0040,  0.0060,  0.0100,  0.0140],
        [ 0.0050,  0.0075,  0.0125,  0.0175],
        [ 0.0060,  0.0090,  0.0150,  0.0210],
        [ 0.0070,  0.0105,  0.0175,  0.0245]], dtype=torch.float64,
       requires_grad=True)
--------------------------------------------------------------------------------
step  1
loss = 9.359808
W2 (W2 is the linear mapping matrix) =
Parameter containing:
tensor([[-0.0132, -0.0199, -0.0331, -0.0464],
        [-0.0116, -0.0174, -0.0290, -0.0406],
        [-0.0099, -0.0149, -0.0248, -0.0348],
        [-0.0083, -0.0124, -0.0207, -0.0290],
        [-0.0066, -0.0099, -0.0166, -0.0232],
        [-0.0050, -0.0075, -0.0124, -0.0174],
        [-0.0033, -0.0050, -0.0083, -0.0116],
        [-0.0017, -0.0025, -0.0041, -0.0058],
        [ 0.0000,  0.0000,  0.0000,  0.0000],
        [ 0.0017,  0.0025,  0.0041,  0.0058],
        [ 0.0033,  0.0050,  0.0083,  0.0116],
        [ 0.0050,  0.0075,  0.0124,  0.0174],
        [ 0.0066,  0.0099,  0.0166,  0.0232],
        [ 0.0083,  0.0124,  0.0207,  0.0290],
        [ 0.0099,  0.0149,  0.0248,  0.0348],
        [ 0.0116,  0.0174,  0.0290,  0.0406]], dtype=torch.float64,
       requires_grad=True)
--------------------------------------------------------------------------------
step  2
loss = 4.027862335488
W2 (W2 is the linear mapping matrix) =
Parameter containing:
tensor([[-0.0167, -0.0250, -0.0417, -0.0584],
        [-0.0146, -0.0219, -0.0365, -0.0511],
        [-0.0125, -0.0188, -0.0313, -0.0438],
        [-0.0104, -0.0156, -0.0261, -0.0365],
        [-0.0083, -0.0125, -0.0209, -0.0292],
        [-0.0063, -0.0094, -0.0156, -0.0219],
        [-0.0042, -0.0063, -0.0104, -0.0146],
        [-0.0021, -0.0031, -0.0052, -0.0073],
        [ 0.0000,  0.0000,  0.0000,  0.0000],
        [ 0.0021,  0.0031,  0.0052,  0.0073],
        [ 0.0042,  0.0063,  0.0104,  0.0146],
        [ 0.0063,  0.0094,  0.0156,  0.0219],
        [ 0.0083,  0.0125,  0.0209,  0.0292],
        [ 0.0104,  0.0156,  0.0261,  0.0365],
        [ 0.0125,  0.0188,  0.0313,  0.0438],
        [ 0.0146,  0.0219,  0.0365,  0.0511]], dtype=torch.float64,
       requires_grad=True)
--------------------------------------------------------------------------------
step  3
loss = 1.733334166004564
W2 (W2 is the linear mapping matrix) =
Parameter containing:
tensor([[-0.0189, -0.0284, -0.0474, -0.0663],
        [-0.0166, -0.0249, -0.0415, -0.0580],
        [-0.0142, -0.0213, -0.0355, -0.0497],
        [-0.0118, -0.0178, -0.0296, -0.0415],
        [-0.0095, -0.0142, -0.0237, -0.0332],
        [-0.0071, -0.0107, -0.0178, -0.0249],
        [-0.0047, -0.0071, -0.0118, -0.0166],
        [-0.0024, -0.0036, -0.0059, -0.0083],
        [ 0.0000,  0.0000,  0.0000,  0.0000],
        [ 0.0024,  0.0036,  0.0059,  0.0083],
        [ 0.0047,  0.0071,  0.0118,  0.0166],
        [ 0.0071,  0.0107,  0.0178,  0.0249],
        [ 0.0095,  0.0142,  0.0237,  0.0332],
        [ 0.0118,  0.0178,  0.0296,  0.0415],
        [ 0.0142,  0.0213,  0.0355,  0.0497],
        [ 0.0166,  0.0249,  0.0415,  0.0580]], dtype=torch.float64,
       requires_grad=True)
--------------------------------------------------------------------------------
step  4
loss = 0.7459160916617404
W2 (W2 is the linear mapping matrix) =
Parameter containing:
tensor([[-0.0204, -0.0306, -0.0511, -0.0715],
        [-0.0179, -0.0268, -0.0447, -0.0626],
        [-0.0153, -0.0230, -0.0383, -0.0536],
        [-0.0128, -0.0192, -0.0319, -0.0447],
        [-0.0102, -0.0153, -0.0255, -0.0358],
        [-0.0077, -0.0115, -0.0192, -0.0268],
        [-0.0051, -0.0077, -0.0128, -0.0179],
        [-0.0026, -0.0038, -0.0064, -0.0089],
        [ 0.0000,  0.0000,  0.0000,  0.0000],
        [ 0.0026,  0.0038,  0.0064,  0.0089],
        [ 0.0051,  0.0077,  0.0128,  0.0179],
        [ 0.0077,  0.0115,  0.0192,  0.0268],
        [ 0.0102,  0.0153,  0.0255,  0.0358],
        [ 0.0128,  0.0192,  0.0319,  0.0447],
        [ 0.0153,  0.0230,  0.0383,  0.0536],
        [ 0.0179,  0.0268,  0.0447,  0.0626]], dtype=torch.float64,
       requires_grad=True)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment