Multi-Model Testing with Sci-kit Learn

mmtest.py

import os
import itertools
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
import matplotlib.pyplot as plt

def Multi_KNN_Class(X, y, test_s=0.1, neighbors=[1,2,3], p_val=[1,2], leaf=[30], iterations=20, fig_s=(15,9), path=os.getcwd(), plot=False, verbose=True):
  """test out all combinations of hyperparameters to find the best model configuration. Returns statistics for mean and standard
  deviation of accuracy over the amount of iterations for each hyperparameter settings."""
    mu_sigma_list = []
    for s in list(set(itertools.product(neighbors, p_val, leaf))):
        i, j, k = s
        acc_list = []
        knn = KNeighborsClassifier(n_neighbors=i, p=j, leaf_size=k)
        for r in range(iterations):
            X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=test_s)
            knn.fit(X_train, y_train)
            acc = knn.score(X_val, y_val)
            acc_list.append(acc)
            
        mu = np.mean(acc_list)
        sigma = np.std(acc_list)
        mu_sigma_list.append(('{}_NN__p_{}__leafsize_{}'.format(i,j,k), mu, sigma))
        if verbose: print('{}_NN__p_{}__leafsize_{}'.format(i,j,k), mu, sigma)
        
        if plot:
            x_axis = list(range(len(acc_list)))
            plt.figure(figsize = fig_s)
            text = 'Accuracy: {}_NN__p_{}__leafsize_{}'.format(i,j,k)
            plt.title(text)
            plt.plot(x_axis, acc_list)
            plt.save_fig(path + '/{}.png'.format(text))
    return mu_sigma_list
    

See: https://www.kaggle.com/limitpointinf0/knn-hyper-parameter-optimization
for example of implementation