Skip to content

Instantly share code, notes, and snippets.

@statcompute
Created November 25, 2018 15:11
Show Gist options
  • Save statcompute/25e99624c7c4ded79e08a82f5978b95f to your computer and use it in GitHub Desktop.
Save statcompute/25e99624c7c4ded79e08a82f5978b95f to your computer and use it in GitHub Desktop.
Dropout Regularization in Deep Neural Networks
from pandas import read_csv, DataFrame
from numpy.random import seed
from sklearn.preprocessing import scale
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from keras.models import Sequential
from keras.constraints import maxnorm
from keras.optimizers import SGD
from keras.layers import Dense, Dropout
from multiprocessing import Pool, cpu_count
from itertools import product
from parmap import starmap
df = read_csv("credit_count.txt")
Y = df[df.CARDHLDR == 1].DEFAULT
X = df[df.CARDHLDR == 1][['AGE', 'ADEPCNT', 'MAJORDRG', 'MINORDRG', 'INCOME', 'OWNRENT', 'SELFEMPL']]
sX = scale(X)
ncol = sX.shape[1]
x_train, x_test, y_train, y_test = train_test_split(sX, Y, train_size = 0.5, random_state = seed(2017))
def tune_dropout(rate1, rate2):
net = Sequential()
## DROPOUT AT THE INPUT LAYER
net.add(Dropout(rate1, input_shape = (ncol,)))
## DROPOUT AT THE 1ST HIDDEN LAYER
net.add(Dense(ncol, init = 'normal', activation = 'relu', W_constraint = maxnorm(4)))
net.add(Dropout(rate2))
## DROPOUT AT THE 2ND HIDDER LAYER
net.add(Dense(ncol, init = 'normal', activation = 'relu', W_constraint = maxnorm(4)))
net.add(Dropout(rate2))
net.add(Dense(1, init = 'normal', activation = 'sigmoid'))
sgd = SGD(lr = 0.1, momentum = 0.9, decay = 0, nesterov = False)
net.compile(loss='binary_crossentropy', optimizer = sgd, metrics = ['accuracy'])
net.fit(x_train, y_train, batch_size = 200, nb_epoch = 50, verbose = 0)
print rate1, rate2, "{:6.4f}".format(roc_auc_score(y_test, net.predict(x_test)))
input_dp = [0.1, 0.2, 0.3]
hidden_dp = [0.2, 0.3, 0.4, 0.5]
parms = [i for i in product(input_dp, hidden_dp)]
seed(2017)
starmap(tune_dropout, parms, pool = Pool(processes = cpu_count()))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment