keras_multiclass_stratified_gridsearch.py

# MLP for Digits Dataset with cross validation
from keras.models import Sequential, model_from_json
from keras.layers import Dense, Dropout
from keras.callbacks import ModelCheckpoint
from keras.constraints import maxnorm
from keras.wrappers.scikit_learn import KerasClassifier
from keras.utils import np_utils
from keras import backend as K
from keras.callbacks import TensorBoard
from keras.optimizers import SGD
from sklearn.model_selection import KFold, GridSearchCV, StratifiedKFold
import numpy as np
import matplotlib.pyplot as plt
import os
import time
import gc

# Silence Warnings
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

tbCallback = TensorBoard(log_dir='../logs', histogram_freq=0, write_graph=True, write_images=False, embeddings_freq=0,
                         embeddings_layer_names=None, embeddings_metadata=None)

# fix random seed for reproducibility
seed = 42
np.random.seed(seed)

# load dataset
X_d = np.loadtxt('../datasets/mixed_extracted/digits_features.csv', delimiter=',')
y_d = np.loadtxt('../datasets/mixed_extracted/digits_target.csv', delimiter=',')

# convert integers to dummy variables (i.e. one hot encoded)
dummy_y = np_utils.to_categorical(y_d)

# define network parameters
num_inputs = X_d.shape[1]
num_classes = dummy_y.shape[1]


# define baseline model
def create_model(learn_rate=0.01, momentum=0, activation='relu', dropout_rate=0.0, weight_constraint=0, neurons=1):
    # create model
    model = Sequential()
    # model without dropout
    # model.add(Dense(37, input_dim=num_inputs, kernel_initializer='normal', activation='relu'))
    # model.add(Dense(20, kernel_initializer='normal', activation='relu'))
    # model.add(Dense(num_classes, kernel_initializer='normal', activation='softmax'))

    # model with dropout on the input and a kernel constraint
    # model.add(Dropout(0.2, input_shape=(num_inputs,)))
    # model.add(Dense(37, kernel_initializer='normal', activation='relu', kernel_constraint=maxnorm(3)))
    # model.add(Dense(20, kernel_initializer='normal', activation='relu', kernel_constraint=maxnorm(3)))
    # model.add(Dense(num_classes, kernel_initializer='normal', activation='softmax'))

    # model with dropout and a kernel constraint on the hidden layers
    # neurons 37, 20
    model.add(Dense(neurons, input_dim=num_inputs, kernel_initializer='normal', activation=activation,
                    kernel_constraint=maxnorm(weight_constraint)))
    model.add(Dropout(0.2))
    model.add(Dense(neurons, kernel_initializer='normal', activation=activation))
    model.add(Dropout(0.2))
    model.add(Dense(num_classes, kernel_initializer='normal', activation='softmax'))

    # Compile model
    optimizer = SGD(lr=learn_rate, momentum=momentum)
    model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])

    return model

# create model
model = KerasClassifier(build_fn=create_model, verbose=0)

# define k-fold cross validation test harness
num_folds = 10
kfold = KFold(n_splits=num_folds, shuffle=True, random_state=seed)
# kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)

# define hyper-parameters for gridsearch
param_grid = dict(activation=['softmax', 'softplus', 'softsign', 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'],
                  learn_rate=[0.001, 0.01, 0.1, 0.2, 0.3],
                  momentum=[0.0, 0.2, 0.4, 0.6, 0.8, 0.9],
                  batch_size=[10, 20, 40, 60, 80, 100],
                  epochs=[10, 50, 100],
                  weight_constraint=[1, 2, 3, 4, 5],
                  dropout_rate=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],
                  neurons=[1, 5, 10, 15, 20, 25, 30]
                  )

print("[INFO] Starting training digits")
print("[INFO] Tuning hyper-parameters for accuracy")
grid = GridSearchCV(estimator=model, param_grid=param_grid, cv=kfold)
start = time.time()
grid_result = grid.fit(X_d, dummy_y)
print("[INFO] GridSearch took {:.2f} seconds".format(time.time() - start))

# summarize results
print("[INFO] GridSearch best score %f using parameters: %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
print("[INFO] Grid scores on development set:")
for mean, stdev, param in zip(means, stds, params):
    print("%f (%f) with: %r" % (mean, stdev, param))

# serialize model to JSON
model_json = model.to_json()
with open("model.json", "w") as json_file:
    json_file.write(model_json)
# serialize weights to HDF5
model.save_weights("model.h5")
print("Saved model to disk")

# later...

# load json and create model
# json_file = open('model.json', 'r')
# loaded_model_json = json_file.read()
# json_file.close()
# loaded_model = model_from_json(loaded_model_json)
# # load weights into new model
# loaded_model.load_weights("model.h5")
# print("Loaded model from disk")

# evaluate loaded model on test data
# loaded_model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
# score = loaded_model.evaluate(X, Y, verbose=0)
# print "%s: %.2f%%" % (loaded_model.metrics_names[1], score[1]*100)

gc.collect()
K.clear_session()