MinliangLin

2025.09.14: Added Newton method and quadratic function.

Visualization of different optimization algorithms used in deep learning.

Click anywhere on the function heatmap to start a minimization. You can toggle the different algorithms (SGD, Momentum, RMSProp, Adam) by clicking on the circles in the lower bar.

The global minimum is on the left. A local minimum is found on the right.

Interestingly, different initializations make some algorithms converge to the local minimum while others converge to the global minimum.

MinliangLin

#!/usr/bin/env bash
###############################################################################
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# This file is licensed under the Apache License, Version 2.0 (the "License").
#
# You may not use this file except in compliance with the License. A copy of
# the License is located at http://aws.amazon.com/apache2.0/.
#
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the

MinliangLin

import numpy as np
h,w = 4,5
a = np.arange(1,n*m+1).reshape(n,m)
print(a)
# [[ 1  2  3  4  5]
#  [ 6  7  8  9 10]
#  [11 12 13 14 15]
#  [16 17 18 19 20]]

def f(a):

MinliangLin

tell application "QuickTime Player"
	#activate
	try
		tell document 1
			set jumpTime to current time
			log "v1 " & jumpTime
		end tell
		tell document 2
			if jumpTime > duration then
				set current time to duration

MinliangLin

import numpy as np
import av

def to_array(plane):
    return np.frombuffer(plane, dtype=np.uint16) \
        .reshape(-1, plane.width)

out = av.open('test-out.mov', mode='w')
ss = out.add_stream("prores", rate=25)
ss.width, ss.height = 1092, 1080

MinliangLin

# from imshowtools import imshow
from PIL import Image
from glob import glob
g = glob('/Users/minliang/Downloads/video_tracking_dataset/BlurFace/img/*.jpg')[:5]
f, axs = plt.subplots(2, 3)
plt.setp(axs, xticks=[], yticks=[])
f.tight_layout(pad=0)
for x, p in zip(axs.flat, g):
    img = Image.open(p)
    x.imshow(img)

MinliangLin

import pandas as pd
import implicit
from scipy.sparse import csr_matrix

df = pd.read_csv("/home/ec2-user/als/ratings.csv")
alpha = 2.0
df['confidence'] = 1.0 + alpha*df.rating

uid_dc = {k:i for i,k in enumerate(set(df.userId))}
tid_dc = {k:i for i,k in enumerate(set(df.movieId))}

MinliangLin

import numpy as np
from scipy.sparse.linalg import cg
from scipy.sparse import random

dim = 100
Y = np.random.randn(dim, dim)
# Y = random(dim, dim) # much harder, Y.nnz = 100, A.nnz = 165
A = Y.T@Y
b = np.random.randn(dim)

MinliangLin

import dask.dataframe as dd
import pandas as pd
import numpy as np
import fasttext

df = pd.DataFrame(np.random.randn(100,4), columns=list('ABCD'))
df2 = dd.from_pandas(df, npartitions=2)
## also works for file
# df.to_csv('data.csv', index=False)
# df2 = dd.read_csv('data.csv', blocksize=500)

MinliangLin

import multiprocessing
print('start')

def run_mp(fn, nproc, method='spawn'):
    mp = multiprocessing.get_context(method)
    ps = []
    for i in range(nproc):
        p = mp.Process(target=fn, args=(i,))
        p.start()
        ps.append(p)