Grid search implementation.

gistfile1.py

def grid_search(features, results):
    gamma_values = [(2 ** n) for n in range(-15, 4)]
    cost_values = [(2 ** n) for n in range(-5, 16)]

    params_values = [(gamma, cost) for gamma in gamma_values
                                   for cost in cost_values]

    best_auc = 0
    best_gamma = None
    best_cost = None

    for (gamma, cost) in params_values:

        folds = Lcv.StratifiedKFold(results, 10)

        auc_sum = 0
        fold_count = 0

        for (train_indices, test_indices) in folds:
            classifier = Ls.SVC(kernel='rbf', gamma=gamma,
                                C=cost, probability=True)
            classifier.fit(features[train_indices], results[train_indices])

            probas = classifier.predict_proba(features[test_indices])

            fpr, tpr, thresholds = Lm.roc_curve(results[test_indices],
                                                probas[:, 1])

            auc = Lm.auc(fpr, tpr)
            auc_sum += auc
            fold_count += 1

        average_auc = auc_sum / fold_count
        if average_auc > best_auc:
            print("AUC improved to", average_auc, "with gamma", gamma,
                  "and cost", cost)
            best_auc = average_auc
            best_gamma = gamma
            best_cost = cost

    print("Best Gamma:", best_gamma)
    print("Best Cost:", best_cost)
    print("Best AUC:", best_auc)

    return {"gamma": best_gamma, "C": best_cost}


def plot_roc(features, results, params):
    print("Using gamma", params["gamma"], "and C", params["C"])

    folds = Lcv.StratifiedKFold(results, 10)

    auc_sum = 0
    fold_count = 0

    pl.clf()
    pl.xlim([0.0, 1.0])
    pl.ylim([0.0, 1.0])
    pl.xlabel('False Positive Rate')
    pl.ylabel('True Positive Rate')
    pl.plot([0, 1], [0, 1], 'k--', label="Coin flip")

    for (train_indices, test_indices) in folds:
        classifier = Ls.SVC(kernel='rbf', gamma=params["gamma"],
                            C=params["C"], probability=True)
        classifier.fit(features[train_indices], results[train_indices])

        probas = classifier.predict_proba(features[test_indices])

        debug(probas)

        fpr, tpr, thresholds = Lm.roc_curve(results[test_indices],
                                            probas[:, 1])

        auc = Lm.auc(fpr, tpr)
        debug(auc)
        auc_sum += auc
        fold_count += 1

        pl.plot(fpr, tpr, label="ROC curve (area = %0.2f)" % auc)

    average_auc = auc_sum / fold_count

    debug("Average:", average_auc)

    pl.legend(loc="lower right")
    pl.title('SVM after PCA (average AUC=%0.2f)' % average_auc)

    pl.show()