Parameters optimization of ImageDataGenerator using randomized search

imagedatagenerator_randomized_search.py

import tensorflow as tf
from tensorflow.keras import layers, models
from keras.callbacks import ReduceLROnPlateau
from keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
import time
import random

# load data
train_data = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
test_data = pd.read_csv("/kaggle/input/digit-recognizer/test.csv")

train_data_y = train_data["label"]
train_data_x = train_data.drop(columns="label")

train_data_x = train_data_x.astype('float64').values.reshape((train_data_len, 28, 28, 1))
test_data = test_data.astype('float64').values.reshape((test_data_len, 28, 28, 1))

# scaling data
train_data_x /= 255.0
test_data /= 255.0

# model creation
def create_model(filters=256, kernel_size=(7, 7), rate=0.4):
    model = models.Sequential()
    
    model.add(layers.Conv2D(filters, kernel_size, activation='relu', padding='same', name='layer1', input_shape=(28, 28, 1)))
    model.add(layers.Dropout(rate))

    model.add(layers.MaxPooling2D((2, 2)))
    model.add(layers.Conv2D(filters*2, (3, 3), activation='relu', name='layer2'))
    model.add(layers.Dropout(rate))

    model.add(layers.MaxPooling2D((2, 2)))
    model.add(layers.Conv2D(filters*2, (3, 3), activation='relu', name='layer3'))
    model.add(layers.Dropout(rate))

    model.add(layers.Flatten())
    model.add(layers.Dense(256, activation='relu'))
    model.add(layers.Dense(10, activation='softmax'))
    
    return model
  
# ----- Randomized search -----
# parameters for fit
bs = 32
ep = 65

# data split
X, X_cv, y, y_cv = train_test_split(train_data_x, train_data_y, test_size=0.2, stratify=train_data_y, random_state=1)

# CNN model parameters
cnn_model_parameters = {'filters':256}

# variables for saving optimum history and minimum val_loss
history_opt_df = None
val_loss_min_opt = 10000.0

# lists for saving each parameter of ImageDataGenerator() and minimum val_loss
rr_l = []
wsr_l = []
hsr_l = []
sr_l = []
zr_l = []
val_loss_min_l = []

# parameters for iteration (not used here)
iter_no = 0

# variables to stop calculation
start_time = time.time()
duration = 7 * 60 * 60 # 7 hours

# parameter list
parameter_name = ["rotation_range", "width_shift_range", "hight_shift_range", "shear_range", "zoom_range"]

while True:
    # print iter no
    print("=========={:2d}==========".format(iter_no))
    
    # create model and compile
    model = create_model(**cnn_model_parameters)
    model.compile(optimizer='adam',
                  loss='sparse_categorical_crossentropy',
                  metrics=['accuracy'])

    # set random parameters for ImageDataGenerator
    rr = random.randint(10, 50)      # 10 - 50
    wsr = random.uniform(0.1, 0.4)   # 0.1 - 0.4
    hsr = random.uniform(0.1, 0.4)   # 0.1 - 0.4
    sr = random.randint(0.1, 0.4)    # 0.1 - 0.4
    zr = random.uniform(0.1, 0.4)    # 0.1 - 0.4
    
    parameter_value = [rr, wsr, hsr, sr, zr]

    for n, v in zip(parameter_name, parameter_value):
        print("{:<20} ; {}".format(n, v))
       
    datagen = ImageDataGenerator(rotation_range=rr,
                                 width_shift_range=wsr,
                                 height_shift_range=hsr,
                                 shear_range=sr,
                                 zoom_range=zr,
                                 fill_mode='nearest')
    
    # callback
    reduce_lr_callback = ReduceLROnPlateau(monitor='val_loss',
                                           factor=0.47,
                                           patience=5,
                                           min_lr=0.00005,
                                           verbose=1)
    # fit
    history1 = model.fit_generator(datagen.flow(X, y, batch_size=bs),
                                   steps_per_epoch=len(X)/bs,
                                   validation_data=(X_cv, y_cv),
                                   epochs=ep,
                                   callbacks=reduce_lr_callback)
    
    # check history and minimum val_loss
    history1_df = pd.DataFrame(history1.history)
    history1_df.to_csv("history_{:02d}.csv".format(iter_no), index=False)
    val_loss_min = history1_df["val_loss"].min()

    # print val_loss_min
    print("{:<20} ; {}".format("minimum val_loss", val_loss_min))
    print("")
    
    # if val_loss is less than before, update history and val_loss
    if val_loss_min < val_loss_min_opt:
        val_loss_min_opt = val_loss_min
        history_opt_df = history1_df
        
    # saving parameters
    rr_l.append(rr)
    wsr_l.append(wsr)
    hsr_l.append(hsr)
    sr_l.append(sr)
    zr_l.append(zr)
    val_loss_min_l.append(val_loss_min)
    
    iter_no += 1
    
    # break if specified time has passed
    if time.time() > start_time + duration:
        break

# ----- save data -----
# save data
history_opt_df.to_csv("history_ImageDataGenerator.csv", index=False)
image_data_generator_results = pd.DataFrame({"rotation_range" : rr_l,
                                             "width_shift_range" : wsr_l,
                                             "hight_shift_range" : hsr_l,
                                             "shear_range" : sr_l,
                                             "zoom_range" : zr_l,
                                             "val_loss" : val_loss_min_l})
image_data_generator_results.to_csv("results_ImageDataGenerator.csv", index=False)

# show parameters of minimum val_loss
index_min = image_data_generator_results["val_loss"].idxmin()
for col in ["rotation_range", "width_shift_range", "hight_shift_range", "shear_range", "zoom_range", "val_loss"]:
    print("{:<20} ; {}".format(col, image_data_generator_results.at[image_data_generator_results.index[index_min], col]))