Batch Gradient Descent.

fitting.py

import matplotlib.pyplot as plt
import numpy as np


class BatchGradientDescent():
    def __init__(self, alpha, model_func=None, deriv_func=None):
        self._alpha = alpha
        self._model_func = model_func
        self._deriv_func = deriv_func

    def fit(self, X, Y, precision=0.00001, iterations=1000):
        self._theta = np.random.random((X.shape[1], 1))
        self.errors = []

        for i in xrange(iterations):
            y_approx = self._model_func(X, self._theta)
            # sum over all errors for batch
            delta = np.sum(self._deriv_func(Y, y_approx))
            self._theta -= self._alpha * delta
            # record error between actual and model
            self.errors.append(-delta)

            if abs(delta) < precision:
                return None

    def predict(self, X):
        return self._model_func(X, self._theta)


def model_function(X, theta):
    return np.dot(X, theta)


def cost_function_derivative(y, y_approx):
    return y_approx - y


def generate_data(sigma=5.0, n=100):
    x = np.arange(n)
    y = x**2 + x + 3  # true function to fit
    y += np.random.randn(y.shape[0]) * sigma
    return (x, y)


if __name__ == "__main__":
    x, y = generate_data(sigma=500.0)

    X = np.array([np.ones(x.shape), x, x**2]).T
    Y = np.array([y]).T

    theta = np.random.random((X.shape[0], 1))

    gd = BatchGradientDescent(alpha=0.000001,
                              model_func=model_function,
                              deriv_func=cost_function_derivative)

    gd.fit(X, Y)
    y_approx = gd.predict(X)

    print "Converged in %d iterations" % len(gd.errors)

    plt.plot(gd.errors)
    plt.show()

    plt.scatter(x, y)
    plt.plot(x, y_approx)
    plt.show()