MNIST.py

MNIST.py

# Setup

import os
import sys
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.preprocessing.image import ImageDataGenerator
from keras.layers.advanced_activations import Quorum
from keras.datasets import mnist
from keras.utils import np_utils
from keras import backend as K
from keras.activations import relu
from keras.callbacks import ModelCheckpoint

af = "pap" #pap, relu, thresh, notrain
init_fn = "he_normal"

def pushln(line):
    sys.stdout.write(line)
    sys.stdout.flush()
    
def step(X):
    return K.switch(X <= 0, 0, 1)

def prepare_input_data(X_train, X_test):
    X_train = X_train.reshape(X_train.shape[0], 1, 28, 28)
    X_test = X_test.reshape(X_test.shape[0], 1, 28, 28)
    X_train = X_train.astype('float32') / 255.0
    X_test = X_test.astype('float32') / 255.0
    return X_train, X_test

def prepare_output_data(y_train, y_test):
    y_train = np_utils.to_categorical(y_train)
    y_test = np_utils.to_categorical(y_test)
    return y_train, y_test

def get_af(name):
    af = name.lower()
    
    if af == "relu":
        return Activation('relu')
    elif af == "pap":
        return Quorum([relu, step])
    elif af == "thresh":
        return Quorum([relu, step], threshold=0.5)
    elif af == "notrain":
        return Quorum([relu, step], trainable=False)
    else:
        raise RuntimeError("Unrecognized activation function: {}".format(name))

def add_convolutional_layers(model, activation_name, n, filter_size, window, 
                             dropout=0.25, stack_finished=True, input_shape=None):
    if input_shape:
        model.add(Convolution2D(filter_size, window, window, border_mode="same", 
                  init=init_fn, input_shape=input_shape))
        model.add(get_af(af))
        n=n-1
        
    for i in range(n):
        model.add(Convolution2D(filter_size, window, window, border_mode="same", init=init_fn))
        model.add(get_af(activation_name))
        
    if stack_finished:
        model.add(MaxPooling2D(pool_size=(2, 2)))
        model.add(Dropout(dropout))
        
# Script

(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train, X_test = prepare_input_data(X_train, X_test)
y_train, y_test = prepare_output_data(y_train, y_test)

model = Sequential()

# 2 x 4 x 3
add_convolutional_layers(model, af, 2, 4, 3, input_shape=(1, 28, 28))

# 2 x 8 x 3
add_convolutional_layers(model, af, 2, 8, 3)
model.add(Dropout(0.25))

# 1 x 10 x FC
model.add(Flatten())
model.add(Dense(10))
model.add(Activation('softmax'))

pushln("Compiling model...")
model.compile(loss='categorical_crossentropy', optimizer='sgd')
pushln("finished.\n")
mc = ModelCheckpoint("mnist_weights_"+af, save_best_only=True, verbose=1)
pushln("Beginning training...\n")
hist = model.fit(X_train, y_train, batch_size=128, verbose=1,
                        nb_epoch=500, show_accuracy=True,
                        validation_data=(X_test, y_test), callbacks=[mc])

Thank you for the code! I just have a question: I wasn't able to find the Quorum activation function in the keras library, did you write it by yourself?