Classification on Madelon dataset, using PCA feature selection

madelon_pca_classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "\n",
    "x_train = pd.read_csv('madelon-pca/data/madelon_train.data', sep=\" \", header=None)\n",
    "y_train = pd.read_csv('madelon-pca/data/madelon_train.labels', sep=\" \", header=None)\n",
    "x_valid = pd.read_csv('madelon-pca/data/madelon_valid.data', sep=\" \", header=None)\n",
    "y_valid = pd.read_csv('madelon-pca/data/madelon_valid.labels', sep=\" \", header=None)\n",
    "\n",
    "x_train.drop(500, axis=1, inplace=True)\n",
    "x_valid.drop(500, axis=1, inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "\n",
    "pca = PCA(.95)\n",
    "dataFull = x_train.append(x_valid)\n",
    "dataFull = pca.fit_transform(dataFull)\n",
    "\n",
    "x_train_pca = dataFull[:x_train.shape[0]]\n",
    "x_valid_pca = dataFull[x_train.shape[0]:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression, SGDClassifier\n",
    "from sklearn.ensemble import ExtraTreesClassifier, GradientBoostingClassifier, RandomForestClassifier\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "\n",
    "models = [\n",
    "    LogisticRegression(),\n",
    "    LogisticRegression(penalty='l1'),\n",
    "    SGDClassifier(),\n",
    "    SGDClassifier(max_iter=1000, tol=1e-3),\n",
    "    ExtraTreesClassifier(),\n",
    "    GradientBoostingClassifier(),\n",
    "    RandomForestClassifier(),\n",
    "    SVC(),\n",
    "    LinearSVC()\n",
    "]\n",
    "\n",
    "names = [\n",
    "    'L2 Logistic Regression', 'L1 Logistic Regression',\n",
    "    'SGDClassifier', 'SGDClassifier Iterated', 'ExtraTreesClassifier',\n",
    "    'GradientBoostingClassifier', 'RandomForestClassifier',\n",
    "    'SVC', 'LinearSVC'\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "\n",
    "# Define a error function\n",
    "def cv_accuracy(model, x, y):\n",
    "    return cross_val_score(model, x, y, scoring=\"accuracy\", cv=5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- L2 Logistic Regression : mean : 0.563500, std : 0.036042\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- L1 Logistic Regression : mean : 0.563500, std : 0.034518\n- SGDClassifier : mean : 0.546500, std : 0.044905\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- SGDClassifier Iterated : mean : 0.537500, std : 0.038046\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- ExtraTreesClassifier : mean : 0.535000, std : 0.034242\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- GradientBoostingClassifier : mean : 0.724000, std : 0.024829\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- RandomForestClassifier : mean : 0.540500, std : 0.016310\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- SVC : mean : 0.500000, std : 0.000000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- LinearSVC : mean : 0.532000, std : 0.018868\n"
     ]
    }
   ],
   "source": [
    "# Perform 5-folds cross-calidation to evaluate the models \n",
    "for model, name in zip(models, names):\n",
    "    # Root mean square error\n",
    "    score = cv_accuracy(model, x_train_pca, y_train)\n",
    "    print(\"- {} : mean : {:.6f}, std : {:4f}\".format(name, score.mean(), score.std()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "\n",
    "class GridSearch:\n",
    "    \n",
    "    def __init__(self, model):\n",
    "        self.model = model\n",
    "    \n",
    "    def grid_get(self, param_grid):\n",
    "        grid_search = GridSearchCV(self.model, param_grid, cv=5, scoring='accuracy', n_jobs=-1)\n",
    "        grid_search.fit(x_train_pca, y_train)\n",
    "        grid_search.cv_results_['mean_test_score'] = grid_search.cv_results_['mean_test_score']\n",
    "        print(pd.DataFrame(grid_search.cv_results_)[['params', 'mean_test_score', 'std_test_score']])\n",
    "        print('\\nBest parameters : {}, best score : {}'.format(grid_search.best_params_, grid_search.best_score_))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                               params  mean_test_score  \\\n0   {'learning_rate': 0.005, 'max_depth': 3, 'n_es...           0.7255   \n1   {'learning_rate': 0.005, 'max_depth': 3, 'n_es...           0.7240   \n2   {'learning_rate': 0.005, 'max_depth': 3, 'n_es...           0.7250   \n3   {'learning_rate': 0.005, 'max_depth': 3, 'n_es...           0.7255   \n4   {'learning_rate': 0.005, 'max_depth': 3, 'n_es...           0.7265   \n5   {'learning_rate': 0.005, 'max_depth': 3, 'n_es...           0.7315   \n6   {'learning_rate': 0.005, 'max_depth': 4, 'n_es...           0.7585   \n7   {'learning_rate': 0.005, 'max_depth': 4, 'n_es...           0.7605   \n8   {'learning_rate': 0.005, 'max_depth': 4, 'n_es...           0.7595   \n9   {'learning_rate': 0.005, 'max_depth': 4, 'n_es...           0.7590   \n10  {'learning_rate': 0.005, 'max_depth': 4, 'n_es...           0.7575   \n11  {'learning_rate': 0.005, 'max_depth': 4, 'n_es...           0.7575   \n12  {'learning_rate': 0.005, 'max_depth': 5, 'n_es...           0.7740   \n13  {'learning_rate': 0.005, 'max_depth': 5, 'n_es...           0.7770   \n14  {'learning_rate': 0.005, 'max_depth': 5, 'n_es...           0.7765   \n15  {'learning_rate': 0.005, 'max_depth': 5, 'n_es...           0.7765   \n16  {'learning_rate': 0.005, 'max_depth': 5, 'n_es...           0.7790   \n17  {'learning_rate': 0.005, 'max_depth': 5, 'n_es...           0.7800   \n18  {'learning_rate': 0.01, 'max_depth': 3, 'n_est...           0.7275   \n19  {'learning_rate': 0.01, 'max_depth': 3, 'n_est...           0.7275   \n20  {'learning_rate': 0.01, 'max_depth': 3, 'n_est...           0.7265   \n21  {'learning_rate': 0.01, 'max_depth': 3, 'n_est...           0.7220   \n22  {'learning_rate': 0.01, 'max_depth': 3, 'n_est...           0.7290   \n23  {'learning_rate': 0.01, 'max_depth': 3, 'n_est...           0.7260   \n24  {'learning_rate': 0.01, 'max_depth': 4, 'n_est...           0.7550   \n25  {'learning_rate': 0.01, 'max_depth': 4, 'n_est...           0.7585   \n26  {'learning_rate': 0.01, 'max_depth': 4, 'n_est...           0.7560   \n27  {'learning_rate': 0.01, 'max_depth': 4, 'n_est...           0.7565   \n28  {'learning_rate': 0.01, 'max_depth': 4, 'n_est...           0.7580   \n29  {'learning_rate': 0.01, 'max_depth': 4, 'n_est...           0.7565   \n..                                                ...              ...   \n42  {'learning_rate': 0.05, 'max_depth': 4, 'n_est...           0.7475   \n43  {'learning_rate': 0.05, 'max_depth': 4, 'n_est...           0.7450   \n44  {'learning_rate': 0.05, 'max_depth': 4, 'n_est...           0.7475   \n45  {'learning_rate': 0.05, 'max_depth': 4, 'n_est...           0.7430   \n46  {'learning_rate': 0.05, 'max_depth': 4, 'n_est...           0.7500   \n47  {'learning_rate': 0.05, 'max_depth': 4, 'n_est...           0.7490   \n48  {'learning_rate': 0.05, 'max_depth': 5, 'n_est...           0.7730   \n49  {'learning_rate': 0.05, 'max_depth': 5, 'n_est...           0.7680   \n50  {'learning_rate': 0.05, 'max_depth': 5, 'n_est...           0.7795   \n51  {'learning_rate': 0.05, 'max_depth': 5, 'n_est...           0.7765   \n52  {'learning_rate': 0.05, 'max_depth': 5, 'n_est...           0.7700   \n53  {'learning_rate': 0.05, 'max_depth': 5, 'n_est...           0.7750   \n54  {'learning_rate': 0.1, 'max_depth': 3, 'n_esti...           0.7130   \n55  {'learning_rate': 0.1, 'max_depth': 3, 'n_esti...           0.7085   \n56  {'learning_rate': 0.1, 'max_depth': 3, 'n_esti...           0.7105   \n57  {'learning_rate': 0.1, 'max_depth': 3, 'n_esti...           0.7085   \n58  {'learning_rate': 0.1, 'max_depth': 3, 'n_esti...           0.7085   \n59  {'learning_rate': 0.1, 'max_depth': 3, 'n_esti...           0.7095   \n60  {'learning_rate': 0.1, 'max_depth': 4, 'n_esti...           0.7440   \n61  {'learning_rate': 0.1, 'max_depth': 4, 'n_esti...           0.7390   \n62  {'learning_rate': 0.1, 'max_depth': 4, 'n_esti...           0.7400   \n63  {'learning_rate': 0.1, 'max_depth': 4, 'n_esti...           0.7385   \n64  {'learning_rate': 0.1, 'max_depth': 4, 'n_esti...           0.7405   \n65  {'learning_rate': 0.1, 'max_depth': 4, 'n_esti...           0.7410   \n66  {'learning_rate': 0.1, 'max_depth': 5, 'n_esti...           0.7675   \n67  {'learning_rate': 0.1, 'max_depth': 5, 'n_esti...           0.7570   \n68  {'learning_rate': 0.1, 'max_depth': 5, 'n_esti...           0.7710   \n69  {'learning_rate': 0.1, 'max_depth': 5, 'n_esti...           0.7670   \n70  {'learning_rate': 0.1, 'max_depth': 5, 'n_esti...           0.7605   \n71  {'learning_rate': 0.1, 'max_depth': 5, 'n_esti...           0.7640   \n\n    std_test_score  \n0         0.024870  \n1         0.024980  \n2         0.024083  \n3         0.023473  \n4         0.022394  \n5         0.022672  \n6         0.024980  \n7         0.025952  \n8         0.027267  \n9         0.028134  \n10        0.029411  \n11        0.029707  \n12        0.018947  \n13        0.016233  \n14        0.016477  \n15        0.016628  \n16        0.018815  \n17        0.018235  \n18        0.021852  \n19        0.021448  \n20        0.026861  \n21        0.025466  \n22        0.019209  \n23        0.020408  \n24        0.021909  \n25        0.025865  \n26        0.028531  \n27        0.024729  \n28        0.027129  \n29        0.019975  \n..             ...  \n42        0.027704  \n43        0.028723  \n44        0.027157  \n45        0.025952  \n46        0.022913  \n47        0.021366  \n48        0.025169  \n49        0.032458  \n50        0.019519  \n51        0.022282  \n52        0.017819  \n53        0.022638  \n54        0.021059  \n55        0.023696  \n56        0.019962  \n57        0.016926  \n58        0.015859  \n59        0.019710  \n60        0.016703  \n61        0.017073  \n62        0.022967  \n63        0.014714  \n64        0.021989  \n65        0.017790  \n66        0.024698  \n67        0.028302  \n68        0.026391  \n69        0.022494  \n70        0.022880  \n71        0.024010  \n\n[72 rows x 3 columns]\n\nBest parameters : {'learning_rate': 0.005, 'max_depth': 5, 'n_estimators': 1000}, best score : 0.78\n"
     ]
    }
   ],
   "source": [
    "GridSearch(GradientBoostingClassifier()).grid_get({\n",
    "    'learning_rate': [0.005, 0.01, .05, 0.1], \n",
    "    'n_estimators': list(range(500, 1001, 100)), \n",
    "    'max_depth': [3, 4, 5]\n",
    "})"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}

Madelon dataset:
http://archive.ics.uci.edu/ml/machine-learning-databases/madelon/