Minimal example Gradient Boosting Regressor using scikit

gradient_boosting.py

import numpy as np
from sklearn.tree import DecisionTreeRegressor

class LeastSquares:
    @staticmethod
    def negative_gradient(preds, y):
        return y - preds

class GradientBoostingRegressor:
    models = []

    def __init__(self, shrinkage=1.0, loss=LeastSquares, tree_params={}, rounds=10):
        self.shrinkage = shrinkage
        self.loss = loss
        self.tree_params = tree_params
        self.rounds = rounds

    def predict(self, X):
        preds = np.zeros(X.shape[0])
        for idx, m in enumerate(self.models):
            preds += self.shrinkage * m.predict(X)
        return preds

    def fit(self, X, y):
        for m in range(self.rounds):
            preds = self.predict(X)
            gradients = self.loss.negative_gradient(preds, y)
            tree = DecisionTreeRegressor(**self.tree_params)
            tree.fit(X, gradients)
            self.models.append(tree)



from sklearn.cross_validation import train_test_split
from sklearn.datasets import load_boston
from sklearn.metrics import mean_absolute_error

boston = load_boston()
X_train, X_test, y_train, y_test = train_test_split(boston.data, boston.target, test_size=0.33, random_state=42)

model = GradientBoostingRegressor(shrinkage=0.1,
                                  loss=LeastSquares,
                                  tree_params={'max_depth':4, 'splitter':'best'},
                                  rounds=200)
model.fit(X_train, y_train)
print(mean_absolute_error(y_test, model.predict(X_test)))