Scaling comparison Ridge vs SGDRegressor(penalty='l2')

ridge_sgd_comparison.py

import numpy as np
from sklearn.linear_model import Ridge, SGDRegressor

rng = np.random.RandomState(42)

n_samples, n_features = 2000, 50

X = rng.randn(n_samples, n_features)
w = rng.randn(n_features)

y_clean = X.dot(w)

noise_level = 1

y_noisy = y_clean + y_clean.std() * rng.randn(n_samples) * noise_level

alpha = 10.

ridge = Ridge(alpha=alpha)
ridge_rescaled = Ridge(alpha=alpha * n_samples)
sgd = SGDRegressor(alpha=alpha, penalty='l2')

ridge.fit(X, y_noisy)
ridge_rescaled.fit(X, y_noisy)
sgd.fit(X, y_noisy)

coef_diff1 = ((ridge.coef_ - sgd.coef_) ** 2).sum()
coef_diff2 = ((ridge_rescaled.coef_ - sgd.coef_) ** 2).sum()

print coef_diff1, coef_diff2