bagging_preds.py

import pandas as pd
import numpy as np
from sklearn.ensemble import RandomForestRegressor
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.datasets import load_boston
sns.set_style("whitegrid")

X, y = load_boston(return_X_y=True)

# Fit Bagging model
model = RandomForestRegressor(
    n_estimators=50, 
    max_features="sqrt", 
    oob_score=True
)

model.fit(X, y)

# Extract inidividual learner predictions
tree_preds = pd.DataFrame([
    i.predict(X) for i in model.estimators_
]).T

def prediction_interval(preds, interval=0.95):
    """Function to extract mean and prediction intervals.
    """
    lb, ub = (1-interval)/2, 1-(1-interval)/2
    p = pd.concat([preds.mean(1), preds.quantile(lb, 1), preds.quantile(ub, 1)], 1)
    p.columns = ["yhat_mean", "yhat_lb", "yhat_ub"]
    return p

yhats = prediction_interval(tree_preds)

error_range = yhats.yhat_ub-yhats.yhat_lb

fig, ax = plt.subplots(figsize=(30, 15))
plt.title("Prediction Intervals", fontsize=(18))
plt.xlabel("Sample", fontsize=(18))
plt.ylabel("Prediction", fontsize=(18))
ax.errorbar(yhats.index, yhats.yhat_mean, yerr=error_range, fmt='ok', ecolor='gray')