optimise hyperparameters

optimisehyperparameters.py

from xgboost import XGBClassifier
from platypus import NSGAII, Problem, Real, Integer
import numpy as np
from sklearn.metrics import accuracy_score, roc_auc_score
from sklearn.model_selection import train_test_split
import sys

X, y = np.arange(10).reshape((5, 2)), range(5)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)

def ml(**kwargs) -> [float, float]:
    learner = XGBClassifier(**kwargs)
    try:
        learner.fit(X_train, y_train)
        y_predict = learner.predict(X_test)
        y_proba = learner.predict_proba(X_test)
        y_pred_prob = np.array(y_proba)
        if len(set(y_test)) == 2 and len(y_pred_prob.shape) > 1:
            y_pred_prob = y_pred_prob[:, 1]
        acc = accuracy_score(y_test, y_predict)
        roc = roc_auc_score(y_test, y_pred_prob, multi_class='ovr'
        return [acc, roc]
    except Exception as e:
        print("error! ", e)
        return [sys.float_info.min, sys.float_info.min]

class HyperTuneProblem(Problem):
    def __init__(self):
        super(HyperTuneProblem, self).__init__(3, 2)
        self.types[:] = [Integer(10, 100), Integer(0, 100), Integer(1, 100)]

    def evaluate(self, solution):
        param_values = solution.variables[:]
        param_names = ["n_estimators", "max_delta_step", "num_parallel_tree"]
        params = dict(zip(param_names, param_values))
        solution.objectives[:] = ml(**params)

# define the range of each parameter, here we are listing the example range for n_estimator, max_delta_step and num_parallel_tree
platypus_parameters = [Integer(10, 100), Integer(0, 100), Integer(1, 100)]

problem = HyperTuneProblem()
problem.types[:] = platypus_parameters
algorithm = NSGAII(problem, population_size=10)
algorithm.run(10)

for solution in algorithm.result:
    print(solution.objectives)