xgboost_skopt.py

from skopt.space import Real, Integer
from skopt.utils import use_named_args
from skopt import gp_minimize

from xgboost import XGBClassifier

from sklearn.model_selection import cross_val_score

cls = XGBClassifier()

space = [
    Integer(4, 12, name='max_depth'),
    Real(10**-5, 10**0, 'log-uniform', name='learning_rate'),
    Integer(5, 200, name='n_estimators'),
    Real(0, 0.5, name='gamma'),
    Integer(1, 5, name='min_child_weight'),
    Real(0.1, 1, name='subsample'),
    Real(0.1, 1, name='colsample_bytree')
]

@use_named_args(space)
def objective(**params):
    cls.set_params(**params)

    return -np.mean(cross_val_score(cls, X, y, cv=5, n_jobs=4,
                                    scoring="neg_mean_absolute_error"))

res_gp = gp_minimize(objective, space, n_calls=50, random_state=0)

"Best score=%.4f" % res_gp.fun