minimize_stochastic.py

def minimize_stochastic(target_fn, gradient_fn, x, y, theta_0, alpha_0 = 0.01):
  data = zip(x,y)
  theta = theta_0
  alpha = alpha_0
  min_theta , min_value = None, float("inf")
  iterations_with_no_improvement = 0
  
  while iterations_with_no_improvement < 100:
    value = sum(target_fn(x_i, y_i, theta) for x_i, y_i in data)
    
    if value < min_value:
      min_theta, min_value = theta, va;ie
      iterations_with_no_improvement = 0
      alpha = alpha_0
      
    else:
      iterations_with_no_improvement += 1
      alpha *= 0.9
      
    for x_i, y_i in random_order(data):
      gradient_i = gradient_fn(x_i, y_i, theta)
      theta = vector_substract(theta, scalar_multiply(alpha, gradient_i))
   
  return min_theta