DIY SGDRegressor

sgdregressor.py

class SGDRegressor(BaseEstimator):
    def __init__(self, eta=10**-3, n_iter=10):
        self.mse_ = []
        self.weights_ = []
        self.eta_ = eta
        self.n_iter_ = n_iter
        
    def fit(self, X, y):
        X = self._prepend_ones(X)
        current_w = np.zeros(X.shape[1], dtype=np.float64, order="C")
        
        for i in range(self.n_iter_):
            current_mse, current_w = self._run_iteration(X, y, current_w)
            self.mse_.append(current_mse)
            self.weights_.append(current_w)
        
        self.w_ = self.weights_[np.argmin(self.mse_)]
        
        return self
    
    def predict(self, X):
        if self.w_ is None:
            raise Exception('You must fit the model first')

        X = self._prepend_ones(X)        
            
        return self._predict(X, self.w_)
    
    def _run_iteration(self, X, y_true, current_w):
        for i, row in enumerate(X):
            current_w = self._calculate_w(row, y_true[i], current_w)
        
        return mean_squared_error(y_true, self._predict(X, current_w)), current_w
            
    def _calculate_w(self, row, y, previous_w):
        current_w = np.zeros(previous_w.shape, dtype=np.float64, order="C")
        tail = self.eta_ * (y - row.dot(previous_w)) 
        
        for i, previous_w_i in enumerate(previous_w):
            current_w[i] = previous_w_i + tail * row[i]
            
        return current_w                  
    
    def _prepend_ones(self, X):
        return np.insert(X, 0, np.ones(X.shape[0], dtype=np.float64, order="C"), axis=1)
    
    def _predict(self, X, w):
        y = np.empty(X.shape[0])
        for i, row in enumerate(X):
            y[i] = w.dot(row)
        
        return y