RFE vs. MutualInfoSelector

feature_selection_rfe.py

import matplotlib.pyplot as plt
import numpy as np
from sklearn.datasets import load_digits
from sklearn.feature_selection import RFE, MutualInfoSelector
from sklearn.preprocessing import minmax_scale
from sklearn.svm import LinearSVC
from sklearn.model_selection import cross_val_score

digits = load_digits()
X = minmax_scale(digits.data)
y = digits.target

svc = LinearSVC()

rfe = RFE(svc, n_features_to_select=1).fit(X, y)
mrmr = MutualInfoSelector(n_features_to_select=X.shape[1],
                          discrete_features=True,
                          discrete_target=True).fit(X, y)
maxrel = MutualInfoSelector(use_redundancy=False,
                            n_features_to_select=X.shape[1],
                            discrete_features=True,
                            discrete_target=True).fit(X, y)

k_all = np.arange(X.shape[1]) + 1
rfe_scores = []
mrmr_scores = []
maxrel_scores = []
for k in k_all:
    mrmr.set_params(n_features_to_select=k)
    maxrel.set_params(n_features_to_select=k)
    X_rfe = X[:, rfe.ranking_ <= k]
    X_mrmr = X[:, mrmr.get_support()]
    X_maxrel = X[:, maxrel.get_support()]

    rfe_scores.append(np.mean(cross_val_score(svc, X_rfe, y, cv=5)))
    mrmr_scores.append(np.mean(cross_val_score(svc, X_mrmr, y, cv=5)))
    maxrel_scores.append(np.mean(cross_val_score(svc, X_maxrel, y, cv=5)))

plt.figure(figsize=(10, 6))
plt.plot(k_all, rfe_scores, '-', label='RFE')
plt.plot(k_all, mrmr_scores, '-', label='mRMR')
plt.plot(k_all, maxrel_scores, '-', label='MaxRel')
plt.xlabel('Number of kept features')
plt.ylabel('5-fold average accuracy')
plt.title('SVC accuracy using different selection methods on digits dataset',
          fontsize=16)
plt.legend(loc='lower right')
plt.show()