STD Loss for DNN

stdloss.py

import sys
import numpy as np

from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.layers.advanced_activations import Quorum
from keras.datasets import mnist
from keras.utils import np_utils
from keras.activations import relu
from keras.layers.advanced_activations import PReLU
from keras import backend as K
from keras.callbacks import Callback

def step(X):
    return K.switch(X <= 0, 0, 1)

(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train = X_train.reshape(X_train.shape[0], 1, 28, 28)
X_test = X_test.reshape(X_test.shape[0], 1, 28, 28)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
Y_train = np_utils.to_categorical(y_train, 10)
Y_test = np_utils.to_categorical(y_test, 10)

def pushstd(string):
    sys.stdout.write(string)
    sys.stdout.flush()
        
def get_activation_layer(name):
    name = name.lower()
    if name == "quorum":
        return Quorum([step, relu])
    elif name == "relu":
        return Activation('relu')
    elif name == "prelu":
        return PReLU()
    else:
        raise RuntimeError("Activation not known: {}".format(name))

def get_new_model(activation_name):
    model = Sequential()
    model.add(Convolution2D(32, 3, 3, border_mode='valid', init='he_normal', input_shape=(1, 28, 28)))
    model.add(get_activation_layer(activation_name))
    model.add(Convolution2D(32, 3, 3))
    model.add(get_activation_layer(activation_name))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))
    model.add(Flatten())
    model.add(Dense(128, init='he_normal'))
    model.add(get_activation_layer(activation_name))
    model.add(Dropout(0.5))
    model.add(Dense(10))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='sgd')
    return model

def test_model(activation_name, nb_epoch):
    model = get_new_model(activation_name)
    hist = model.fit(X_train, Y_train, batch_size=128, nb_epoch=nb_epoch, 
              verbose=1, validation_data=(X_test, Y_test), show_accuracy=True)
    return hist.history

import os
import cPickle
from collections import defaultdict

nb_epoch=50
activations_to_try = ["quorum", "relu", "prelu"]

if os.path.exists("./results.pkl"):
    results = cPickle.load(open("results.pkl"))
    s = len(results[activations_to_try[0]+"-loss"])
else:
    results = defaultdict(list)
    s = 0

while True:
    pushstd("-- Round {} --\n".format(s))
    for activation in activations_to_try:
        pushstd("Processing {}...\n".format(activation))
        hist = test_model(activation, nb_epoch)
        results[activation+'-loss'].append(hist['loss'][-1])
        results[activation+'-val_loss'].append(hist['val_loss'][-1])
        results[activation+'-acc'].append(hist['acc'][-1])
        results[activation+'-val_acc'].append(hist['val_acc'][-1])
    cPickle.dump(results, open("results.pkl", "wb"))
    s=s+1