AUC Score Gist

churn.py

kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=1)    
for train, test in kfold.split(X, y):
    # Initial tests appear to indicate no overfitting, dropout layer unneccesary
    ann = Sequential()    
    ann.add(Dense(500, activation='tanh', kernel_initializer='random_normal', input_shape=(X[train].shape[1],)))
    ann.add(Dropout(rate=0.2))
    ann.add(Dense(1, activation='sigmoid', kernel_initializer='random_normal'))
    ann.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
    
    ann.fit(X[train], y[train], batch_size=250, epochs=150, verbose=0)
    fpr, tpr, _ = roc_curve(y[test],ann.predict_proba(X[test]))
    auc_score = roc_auc_score(y[test], ann.predict(X[test]))
    
    print('AUC Score: {:.3f}%'.format(auc_score))
    print('State Dummy Variables Accuracy: {:.2f}%'.format(ann.evaluate(X[test], y[test], verbose=0)[1]*100))