bench_gp_vs_rscv.py

from sklearn.datasets import make_classification, load_iris, load_digits
from scipy.stats import uniform, rv_continuous
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import RandomizedSearchCV, GridSearchCV, SequentialSearchCV
import numpy as np
import matplotlib.pyplot as plt

# iris = load_iris()
# digits = load_digits()
# X, y = iris.data, iris.target

digits = load_digits()
X, y = digits.data, digits.target

class NotUniform(rv_continuous):
    def __init__(self, a=None, b=None, seed=None):
        self.uniform = uniform(loc=a, scale=b)
        self.uniform.random_state = seed
        super(Uniform, self).__init__(seed=seed)
        
    def _rvs(self):
        return 10**self.uniform.rvs()

clf = LogisticRegression(multi_class='multinomial', solver='newton-cg')
rvs_obj = NotUniform(-5, 10, 0)
sscv = SequentialSearchCV(clf, {'C': rvs_obj}, random_state=0, verbose=1, n_iter=20)
sscv.fit(X, y)

sscv_cum_scores = [np.max(
      [sscv.grid_scores_[j].mean_validation_score for j in range(i)])
      for i in range(1, len(sscv.grid_scores_))]

rvs_obj = NotUniform(-5, 10, 0)
rscv = RandomizedSearchCV(clf, {'C': rvs_obj}, random_state=0, verbose=1, n_iter=20)
rscv.fit(X, y)

rscv_cum_scores = [np.max(
      [rscv.grid_scores_[j].mean_validation_score for j in range(i)])
      for i in range(1, len(rscv.grid_scores_))]

params = {'C': np.logspace(-5, 5, 20)[::-1]}

grid_search = GridSearchCV(clf, params)
grid_search.fit(X, y)

grid_cum_scores = [np.max(
      [grid_search.grid_scores_[j].mean_validation_score for j in range(i)])
      for i in range(1, len(grid_search.grid_scores_))]


plt.plot(rscv_cum_scores, label='random', linestyle='--')
plt.plot(sscv_cum_scores, label='gp', linestyle='--')
plt.plot(grid_cum_scores, label='grid_search', linestyle='--')

plt.legend(loc='lower right')
plt.ylabel('Score (higher is better)')
plt.xlabel('Number of model evaluations')
plt.show()