Notebook for Porto Seguro

PortoSeguro_Standard.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\mdarq\\AppData\\Local\\conda\\conda\\envs\\Standard3_6\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import time\n",
    "\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn import preprocessing\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.preprocessing import LabelBinarizer, StandardScaler\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_cat(data, x_axis, y_axis, hue):\n",
    "    plt.figure()    \n",
    "    sns.barplot(x=x_axis, y=y_axis, hue=hue, data=data)\n",
    "    sns.set_context(\"notebook\", font_scale=1.6)\n",
    "    plt.legend(loc=\"upper right\", fontsize=\"medium\") \n",
    "\n",
    "def plot_correlation_map(df, annotation=True ):\n",
    "    corr = df.corr()\n",
    "    _ , ax = plt.subplots( figsize =( 12 , 10 ) )\n",
    "    cmap = sns.diverging_palette( 220 , 10 , as_cmap = True )\n",
    "    _ = sns.heatmap(\n",
    "        corr, \n",
    "        cmap = cmap,\n",
    "        square=True, \n",
    "        cbar_kws={ 'shrink' : .9 }, \n",
    "        ax=ax, \n",
    "        annot = annotation, \n",
    "        annot_kws = { 'fontsize' : 12 }\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "class encoding:\n",
    "    \"\"\"\n",
    "    This will apply sklearn.preprocessing.LabelBinarizer and sklearn.preprocessing.StandardScaler\n",
    "    :example:\n",
    "    >>> Test\n",
    "    Blabla\n",
    "    >>> Other test\n",
    "    TTT blabla\n",
    "    \n",
    "    :param data: A pandas DataFrame\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        #binary, categorical or numerical features\n",
    "        self.binary = list()\n",
    "        self.categorical = list()\n",
    "        self.numeric = list()\n",
    "        self.feat_remove = list()\n",
    "#        self.data = dat\n",
    "\n",
    "    def __repr__(self):\n",
    "        return \"<Binarizer and Scaler> object\"\n",
    "    \n",
    "    def get_cat(self,dat):\n",
    "        colnames = dat.columns\n",
    "        print(colnames)\n",
    "        for col in colnames:\n",
    "            c = col.split('_')[-1]\n",
    "            if c == \"cat\":\n",
    "                self.categorical.append(col)                                \n",
    "            elif c == \"bin\":\n",
    "                self.binary.append(col)             \n",
    "            elif (c != \"cat\")&(c != \"bin\"):\n",
    "                self.numeric.append(col)\n",
    "        return self.categorical, self.binary, self.numeric\n",
    "    \n",
    "    def NA_encode(self,dat):\n",
    "        #Implement 'most frequent' strategy\n",
    "        print(\"Processing categorical features...\")\n",
    "        for c in self.categorical:\n",
    "            #Null values are represented by \"-1\", switching to NaN\n",
    "            print(\"Number of missing values for feature: {0}\".format(c))\n",
    "            dat.loc[(dat[c] == -1, c)] = np.NaN\n",
    "            na_count = dat[c].isnull().sum()\n",
    "            dat[c].fillna(value=dat[c].mode()[0], inplace=True)\n",
    "            print(na_count) #Checking number of null values            \n",
    "        #Implement 'mean' or 'median\" strategy\n",
    "        print(\"Processing numerical features...\")\n",
    "        for c in self.numeric:\n",
    "            #NaN are represented as \"-1\" in the original data\n",
    "            dat.loc[(dat[c] == -1, c)] = np.NaN\n",
    "            na_count = dat[c].isnull().sum()\n",
    "            print(\"Number of missing values for feature: {0}\\n{1}\".format(c,na_count))\n",
    "            #Replace NaN values\n",
    "            dat[c].fillna(value=dat[c].mean(), inplace=True)\n",
    "\n",
    "    def Binarize_scale(self, dat):\n",
    "        \"\"\"\n",
    "        Get column names and determine if numeric/cat/bin\n",
    "        :param data: A pandas DataFrame\n",
    "        \"\"\"\n",
    "        self.get_cat(dat)\n",
    "        print(\"Encoding null values\")\n",
    "        self.NA_encode(dat)\n",
    "        print(\"Binarizing...\")\n",
    "#       Let's first create the LabelBinarized features    \n",
    "                  \n",
    "        label = LabelBinarizer()\n",
    "        for c in self.categorical:   \n",
    "            _ = label.fit_transform(dat[c])\n",
    "            for i in range(np.shape(_)[1]):              \n",
    "#               self.data[str(c)+\"_\"+str(i)] = _[:,i]\n",
    "               dat[str(c)+\"_\"+str(i)] = _[:,i]\n",
    "        print(\"Scaling...\")\n",
    "#       Scale numeric features\n",
    "        scaler = StandardScaler()\n",
    "        for c in self.numeric:\n",
    "            _ = scaler.fit_transform(np.float64(dat[c]).reshape(-1,1))\n",
    "        print(\"Finished !\")\n",
    "        pass  \n",
    "        return dat\n",
    "\n",
    "def RFmodel(n_estimators,min_samples_leaf):\n",
    "    '''\n",
    "    Defining a random forest classifier, Fitting on training set, Prediction on test set\n",
    "    ---\n",
    "    Returns    \n",
    "    train_score : the accuracy score calculated on the training set\n",
    "    test_score : the accuracy score calculated on the test set\n",
    "    auc_roc : auc_roc estimated on test set\n",
    "    RF_model : the Random Forest model for further prediction\n",
    "    '''\n",
    "    RF_model = RandomForestClassifier(n_jobs=-1,\n",
    "                                      n_estimators=n_estimators,\n",
    "                                      oob_score=False,\n",
    "                                      min_samples_leaf=min_samples_leaf,\n",
    "                                     class_weight={0:0.95, 1:16})\n",
    "    t_init = time.time()\n",
    "    print(\"Fitting train set with \\n n_estimators = {0} \\n min_samples_leaf = {1} \".format(n_estimators, min_samples_leaf))\n",
    "    RF_model.fit(X_train,y_train)\n",
    "    t_final = time.time()\n",
    "    print(\"Total time: {0}s\".format(t_final-t_init))\n",
    "    train_score = RF_model.score(X_train,y_train)\n",
    "    test_score = RF_model.score(X_test,y_test)\n",
    "    print(\"Train score: {0}\".format(train_score))\n",
    "    print(\"Test score: {0}\".format(test_score))\n",
    "    pred_RF_test = RF_model.predict(X_test)\n",
    "    print(pred_RF_test.sum())\n",
    "    auc_roc = roc_auc_score(y_test,pred_RF_test)\n",
    "    print(\"AUC-ROC: {0}\\n\".format(auc_roc))    \n",
    "    return train_score, test_score, auc_roc, RF_model\n",
    "\n",
    "def logitmodel(C=1):\n",
    "    \"\"\"\n",
    "    Trains and fits a logistic regression model\n",
    "    \"\"\"\n",
    "    print(\"Fitting with \\n C = {0}\".format(C))\n",
    "    logit_model = LogisticRegression(n_jobs=-1)\n",
    "    t_init = time.time()\n",
    "    logit_model.fit(X_train, y_train)\n",
    "    t_final = time.time()\n",
    "    print(\"Total time: {0}s\".format(t_final-t_init))\n",
    "    train_score = logit_model.score(X_train,y_train)\n",
    "    test_score = logit_model.score(X_test,y_test)\n",
    "    print(train_score)\n",
    "    print(test_score)\n",
    "    pred_logit_test = logit_model.predict(X_test)\n",
    "    print(pred_logit_test.sum())\n",
    "    auc_roc = roc_auc_score(y_test,pred_logit_test)\n",
    "    print(\"AUC-ROC: {0}\\n\".format(auc_roc))  \n",
    "    return train_score, test_score, auc_roc, logit_model\n",
    "\n",
    "def dropfeatures(dat, feat):\n",
    "    '''\n",
    "    Deletes colums from a DataFrame\n",
    "    parameters\n",
    "    dat = DataFrame \n",
    "    feat = Features to be removed from the DataFrame \n",
    "    '''\n",
    "    dat.drop(feat, inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data exploration"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this competition, we will predict the probability that an auto insurance policy holder files a claim.\n",
    "\n",
    "In the train and test data, features that belong to similar groupings are tagged as such in the feature names (e.g., ind, reg, car, calc). In addition, feature names include the postfix bin to indicate binary features and cat to indicate categorical features. Features without these designations are either continuous or ordinal. Values of -1 indicate that the feature was missing from the observation. The target columns signifies whether or not a claim was filed for that policy holder.\n",
    "\n",
    "We do not have any description of the features so data engineering will be limited"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"train.csv\")\n",
    "data_valid = pd.read_csv(\"test.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Missing data\n",
    "\n",
    "I'll check the number of missing values for each feature and remove those containing too many missing values.\n",
    "Remember that in this dataset, NaN values are represented as \"-1\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ps_car_03_cat    411231\n",
       "ps_car_05_cat    266551\n",
       "ps_reg_03        107772\n",
       "ps_car_14         42620\n",
       "ps_car_07_cat     11489\n",
       "ps_ind_05_cat      5809\n",
       "ps_car_09_cat       569\n",
       "ps_ind_02_cat       216\n",
       "ps_car_01_cat       107\n",
       "ps_ind_04_cat        83\n",
       "ps_car_02_cat         5\n",
       "ps_car_11             5\n",
       "ps_car_12             1\n",
       "ps_ind_03             0\n",
       "ps_ind_18_bin         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(data == -1).sum().sort_values(ascending=False)[:15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ps_car_03_cat    616911\n",
       "ps_car_05_cat    400359\n",
       "ps_reg_03        161684\n",
       "ps_car_14         63805\n",
       "ps_car_07_cat     17331\n",
       "ps_ind_05_cat      8710\n",
       "ps_car_09_cat       877\n",
       "ps_ind_02_cat       307\n",
       "ps_car_01_cat       160\n",
       "ps_ind_04_cat       145\n",
       "ps_car_02_cat         5\n",
       "ps_car_11             1\n",
       "ps_ind_16_bin         0\n",
       "ps_reg_02             0\n",
       "ps_reg_01             0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(data_valid == -1).sum().sort_values(ascending=False)[:15]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I'll drop features \"ps_car_03_cat\", \"ps_car_05_cat\" as well as \"ps_reg_03\" since it has 50% more missing values in the validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_to_drop = [\"ps_car_03_cat\",\"ps_car_05_cat\", \"ps_reg_03\"]\n",
    "dropfeatures(data, feat=features_to_drop)\n",
    "dropfeatures(data_valid, features_to_drop)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data imbalance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21694 claims were filed out of 595212 customers\n",
      "This represents 3.64% of customers\n"
     ]
    }
   ],
   "source": [
    "no_claims_filed = (data[\"target\"]==0).sum()\n",
    "claims_filed = (data[\"target\"]==1).sum()\n",
    "print(\"{0} claims were filed out of {1} customers\".format(claims_filed,claims_filed+no_claims_filed))\n",
    "print(\"This represents {:.2f}% of customers\".format(100*(claims_filed/(claims_filed+no_claims_filed))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1ad825f35c0>"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD9CAYAAABA8iukAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAFShJREFUeJzt3H+w5XV93/HnS5ZfGmD5sTBkFwLV\nTSuSgnAL2zpJrXRgIRmXTCDBUXdlaHfCoDW1acROEyxoqxOnKKOSkIAs0RYoBtlYdN0iaJrh1yLI\nz+DeooE7S2FxgZAyaoB3/zif1cPl3Hs/u7Cei/t8zJw53+/7+/5+P9+ze+593e+Pc1JVSJLU4zXj\n3gFJ0quHoSFJ6mZoSJK6GRqSpG6GhiSpm6EhSerWFRpJFia5JslfJ3kgyT9Nsl+S9Uk2tud9W2+S\nXJRkMsndSY4Z2s6q1r8xyaqh+rFJ7mnrXJQkrT5yDEnSePQeaXwK+GpV/SPgKOAB4FzghqpaCtzQ\n5gFOBpa2x2rgYhgEAHAecDxwHHDeUAhc3Hq3rre81WcaQ5I0BnOGRpK9gV8BLgWoqh9V1VPACmBN\na1sDnNqmVwBX1MAtwMIkBwMnAeuraktVPQmsB5a3ZXtX1c01+KThFdO2NWoMSdIYLOjo+QfAZuBz\nSY4C7gDeDxxUVY8CVNWjSQ5s/YuBR4bWn2q12epTI+rMMsaMDjjggDrssMM6XpYkaas77rjjiapa\nNFdfT2gsAI4B3ldVtyb5FLOfJsqIWm1HvVuS1QxOb3HooYeyYcOGbVldknZ6Sf6mp6/nmsYUMFVV\nt7b5axiEyGPt1BLt+fGh/kOG1l8CbJqjvmREnVnGeJGquqSqJqpqYtGiOYNSkrSd5gyNqvq/wCNJ\n/mErnQDcD6wFtt4BtQq4rk2vBVa2u6iWAU+3U0zrgBOT7NsugJ8IrGvLnkmyrN01tXLatkaNIUka\ng57TUwDvA76QZDfgIeBMBoFzdZKzgIeB01vv9cApwCTwbOulqrYkuQC4vfWdX1Vb2vTZwOXAnsBX\n2gPgYzOMIUkag/ysfTX6xMREeU1DkrZNkjuqamKuPj8RLknqZmhIkroZGpKkboaGJKmboSFJ6tZ7\ny+1O5dh/f8W4d0HzzB1/uHLcuyDNCx5pSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuh\nIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuh\nIUnqZmhIkroZGpKkbl2hkeR7Se5JcleSDa22X5L1STa2531bPUkuSjKZ5O4kxwxtZ1Xr35hk1VD9\n2Lb9ybZuZhtDkjQe23Kk8S+q6uiqmmjz5wI3VNVS4IY2D3AysLQ9VgMXwyAAgPOA44HjgPOGQuDi\n1rt1veVzjCFJGoOXc3pqBbCmTa8BTh2qX1EDtwALkxwMnASsr6otVfUksB5Y3pbtXVU3V1UBV0zb\n1qgxJElj0BsaBXwtyR1JVrfaQVX1KEB7PrDVFwOPDK071Wqz1adG1Gcb40WSrE6yIcmGzZs3d74k\nSdK2WtDZ95aq2pTkQGB9kr+epTcjarUd9W5VdQlwCcDExMQ2rStJ6td1pFFVm9rz48C1DK5JPNZO\nLdGeH2/tU8AhQ6svATbNUV8yos4sY0iSxmDO0EjyuiR7bZ0GTgTuBdYCW++AWgVc16bXAivbXVTL\ngKfbqaV1wIlJ9m0XwE8E1rVlzyRZ1u6aWjltW6PGkCSNQc/pqYOAa9tdsAuA/1ZVX01yO3B1krOA\nh4HTW//1wCnAJPAscCZAVW1JcgFwe+s7v6q2tOmzgcuBPYGvtAfAx2YYQ5I0BnOGRlU9BBw1ov59\n4IQR9QLOmWFblwGXjahvAI7sHUOSNB5+IlyS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEnd\nDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEnd\nDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd26QyPJLknuTPLlNn94kluTbExy\nVZLdWn33Nj/Zlh82tI0PtfqDSU4aqi9vtckk5w7VR44hSRqPbTnSeD/wwND8x4ELq2op8CRwVquf\nBTxZVW8ALmx9JDkCOAN4E7Ac+GwLol2AzwAnA0cA72i9s40hSRqDrtBIsgT4VeBP23yAtwHXtJY1\nwKltekWbpy0/ofWvAK6sqh9W1XeBSeC49pisqoeq6kfAlcCKOcaQJI1B75HGJ4HfA15o8/sDT1XV\nc21+CljcphcDjwC05U+3/h/Xp60zU322MV4kyeokG5Js2Lx5c+dLkiRtqzlDI8mvAY9X1R3D5RGt\nNceyV6r+0mLVJVU1UVUTixYtGtUiSXoFLOjoeQvw9iSnAHsAezM48liYZEE7ElgCbGr9U8AhwFSS\nBcA+wJah+lbD64yqPzHLGJKkMZjzSKOqPlRVS6rqMAYXsr9eVe8EbgROa22rgOva9No2T1v+9aqq\nVj+j3V11OLAUuA24HVja7pTarY2xtq0z0xiSpDF4OZ/T+CDwgSSTDK4/XNrqlwL7t/oHgHMBquo+\n4GrgfuCrwDlV9Xw7ingvsI7B3VlXt97ZxpAkjUHP6akfq6qbgJva9EMM7nya3vMD4PQZ1v8o8NER\n9euB60fUR44hSRoPPxEuSepmaEiSuhkakqRuhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZo\nSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZo\nSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuc4ZGkj2S3Jbk20nuS/KfWv3wJLcm2ZjkqiS7tfru\nbX6yLT9saFsfavUHk5w0VF/eapNJzh2qjxxDkjQePUcaPwTeVlVHAUcDy5MsAz4OXFhVS4EngbNa\n/1nAk1X1BuDC1keSI4AzgDcBy4HPJtklyS7AZ4CTgSOAd7ReZhlDkjQGc4ZGDfxdm921PQp4G3BN\nq68BTm3TK9o8bfkJSdLqV1bVD6vqu8AkcFx7TFbVQ1X1I+BKYEVbZ6YxJElj0HVNox0R3AU8DqwH\n/g/wVFU911qmgMVtejHwCEBb/jSw/3B92joz1fefZQxJ0hh0hUZVPV9VRwNLGBwZvHFUW3vODMte\nqfpLJFmdZEOSDZs3bx7VIkl6BWzT3VNV9RRwE7AMWJhkQVu0BNjUpqeAQwDa8n2ALcP1aevMVH9i\nljGm79clVTVRVROLFi3alpckSdoGPXdPLUqysE3vCfxL4AHgRuC01rYKuK5Nr23ztOVfr6pq9TPa\n3VWHA0uB24DbgaXtTqndGFwsX9vWmWkMSdIYLJi7hYOBNe0up9cAV1fVl5PcD1yZ5CPAncClrf9S\n4M+STDI4wjgDoKruS3I1cD/wHHBOVT0PkOS9wDpgF+CyqrqvbeuDM4whSRqDOUOjqu4G3jyi/hCD\n6xvT6z8ATp9hWx8FPjqifj1wfe8YkqTx8BPhkqRuhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ\n6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ\n6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnqNmdoJDkkyY1JHkhyX5L3t/p+\nSdYn2die9231JLkoyWSSu5McM7StVa1/Y5JVQ/Vjk9zT1rkoSWYbQ5I0Hj1HGs8B/66q3ggsA85J\ncgRwLnBDVS0FbmjzACcDS9tjNXAxDAIAOA84HjgOOG8oBC5uvVvXW97qM40hSRqDOUOjqh6tqm+1\n6WeAB4DFwApgTWtbA5zaplcAV9TALcDCJAcDJwHrq2pLVT0JrAeWt2V7V9XNVVXAFdO2NWoMSdIY\nbNM1jSSHAW8GbgUOqqpHYRAswIGtbTHwyNBqU602W31qRJ1ZxpAkjUF3aCT5OeCLwO9U1d/O1jqi\nVttR75ZkdZINSTZs3rx5W1aVJG2DrtBIsiuDwPhCVf15Kz/WTi3Rnh9v9SngkKHVlwCb5qgvGVGf\nbYwXqapLqmqiqiYWLVrU85IkSduh5+6pAJcCD1TVfx1atBbYegfUKuC6ofrKdhfVMuDpdmppHXBi\nkn3bBfATgXVt2TNJlrWxVk7b1qgxJEljsKCj5y3Au4F7ktzVav8B+BhwdZKzgIeB09uy64FTgEng\nWeBMgKrakuQC4PbWd35VbWnTZwOXA3sCX2kPZhlDkjQGc4ZGVf1vRl93ADhhRH8B58ywrcuAy0bU\nNwBHjqh/f9QYkqTx8BPhkqRuhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6G\nhiSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6G\nhiSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnqNmdoJLksyeNJ7h2q7ZdkfZKN7XnfVk+Si5JMJrk7\nyTFD66xq/RuTrBqqH5vknrbORUky2xiSpPHpOdK4HFg+rXYucENVLQVuaPMAJwNL22M1cDEMAgA4\nDzgeOA44bygELm69W9dbPscYkqQxmTM0quqbwJZp5RXAmja9Bjh1qH5FDdwCLExyMHASsL6qtlTV\nk8B6YHlbtndV3VxVBVwxbVujxpAkjcn2XtM4qKoeBWjPB7b6YuCRob6pVputPjWiPtsYkqQxeaUv\nhGdErbajvm2DJquTbEiyYfPmzdu6uiSp0/aGxmPt1BLt+fFWnwIOGepbAmyao75kRH22MV6iqi6p\nqomqmli0aNF2viRJ0ly2NzTWAlvvgFoFXDdUX9nuoloGPN1OLa0DTkyyb7sAfiKwri17JsmydtfU\nymnbGjWGJGlMFszVkOS/A28FDkgyxeAuqI8BVyc5C3gYOL21Xw+cAkwCzwJnAlTVliQXALe3vvOr\nauvF9bMZ3KG1J/CV9mCWMSRJYzJnaFTVO2ZYdMKI3gLOmWE7lwGXjahvAI4cUf/+qDEkSePjJ8Il\nSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQk\nSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1G3BuHdAUr+H\nz/+lce+C5qFD/+Cen9pYHmlIkroZGpKkboaGJKnbvA+NJMuTPJhkMsm5494fSdqZzevQSLIL8Bng\nZOAI4B1JjhjvXknSzmtehwZwHDBZVQ9V1Y+AK4EVY94nSdppzffQWAw8MjQ/1WqSpDGY75/TyIha\nvaQpWQ2sbrN/l+TBHbpXO5cDgCfGvRPjlk+sGvcu6KV8b2513qhfldvsF3qa5ntoTAGHDM0vATZN\nb6qqS4BLflo7tTNJsqGqJsa9H9J0vjfHY76fnrodWJrk8CS7AWcAa8e8T5K005rXRxpV9VyS9wLr\ngF2Ay6rqvjHvliTttOZ1aABU1fXA9ePej52Yp/00X/neHINUveS6siRJI833axqSpHnE0JAkdTM0\nfsYlOSzJvXP0/HySa3bQ+Lsn+V9J7kryW0n+dOtXwST5XpIDtmFb70ny6R2xn3rlJflwkt+do+e3\nk6zcQeP/cpL72ntv8db3eJK3JvnyNm7rpiTe3sur4EK4dryq2gSctoM2/2Zg16o6us1ftYPG0atQ\nVf3RDtz8O4FPVNXn2vyOeo/vVDzSmCfaEcEDSf6k/XX0tSR7Jjk6yS1J7k5ybZJ9Z9nGG9pf9d9O\n8q0krx8xxl+2Zd9K8s+G6ve26fck+VKSv0jy3STvTfKBJHe2/div9f2bJPe3/bpyhv05EPg8cHT7\na+/1M/3FluRdSW5rfX/cvqySJGcm+U6SbwBv2c5/Xv0UJFnZ3g/fTvJn05b96yS3t2VfTPLaVv/x\n0Uh7b1yY5JvtZ+GfJPnzJBuTfKT1vC7J/2zbuTfJb82wL/8K+E3gD5J8YaYj7ra9y9q+3ZlkRavv\nmeTK9nquAvZ8Rf+xXs2qysc8eACHAc8BR7f5q4F3AXcD/7zVzgc+Ocs2bgV+vU3vAby2bffeVnst\nsEebXgpsGBp7a897gElgL2AR8DTw223ZhcDvtOlNwO5teuEs+/RW4MtD8zcBE236ewy+CuKNwF8w\nOCIB+CywEjgYeLjtx27AXwGfHvf/lY+R/89vAh4EDmjz+wEfBn63ze8/1PsR4H1terjnJuDjbfr9\n7T12MLA7g2+H2B/4DeBPhra1zyz7dDlwWpsefo//+D0J/GfgXW16IfAd4HXABxh8LgzgH7efzYlx\n/zvPh4enp+aX71bVXW36DuD1DH4hf6PV1gD/Y9SKSfYCFlfVtQBV9YNWH27bFfh0kqOB54FfnGE/\nbqyqZ4BnkjzN4Bc6wD0MfoBgEGZfSPIl4Evb9Cpf6gTgWOD2tr97Ao8DxwM3VdXm9lqummWfNV5v\nA66pqicAqmrLtPfeke1oYSHwcww+sDvK1m98uAe4r6oeBUjyEIOvFLoH+ESSjzP4xf+XL3O/TwTe\nPnTtZQ/gUOBXgIvaa7k7yd0vc5yfGYbG/PLDoennGfyA9er5xrJ/CzwGHMXg1OQPOvbjhaH5F/jJ\ne+ZXGfxgvR34/SRvqqrntmF/hwVYU1UfelExOZURX1CpeSnM/n91OXBqVX07yXsY/LU/yvB7bfr7\ncEFVfSfJscApwH9J8rWqOv9l7vdvVNWLvuS0BZ7vvRG8pjG/PQ08meSX2/y7gW+MaqyqvwWm2i/a\nrXctvXZa2z7Ao1X1QtvWLtuzU0leAxxSVTcCv8dP/nrcXjcAp7VrICTZL8kvMDjd9tYk+yfZFTj9\nZYyhHesG4DeT7A+D/8Npy/cCHm3/j+/c3kGS/DzwbFV9HvgEcMz2bqtZB7wvLSWSvLnVv7l1P5Mc\nyU+OsHd6HmnMf6uAP2oB8BBw5iy97wb+OMn5wN8z+CX7wtDyzwJfTHI6cCPw/7Zzn3YBPp9kHwZ/\nqV1YVU9t57aoqvuT/Efgay2Q/h44p6puSfJh4GbgUeBbbGfQaceqqvuSfBT4RpLngTsZXLPa6vcZ\n/BHwNwxOMe21nUP9EvCHSV5g8D45e7t3euAC4JPA3S04vgf8GnAx8Ll2Wuou4LaXOc7PDL9GRJLU\nzdNTkqRunp56FUryGV76mYVP1U8+xPRTl+RMBrdJDvurqjpnHPujnUeSa4HDp5U/WFUz3aGll8HT\nU5Kkbp6ekiR1MzQkSd0MDUlSN0NDktTN0JAkdfv/r38CyaZ1tKIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1ad80894da0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "sns.barplot(x=[\"no_claims_filed\", \"claims_filed\"], y=[no_claims_filed,claims_filed])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data is rather imbalanced, so for the models to predict positive values, we'll have to balance it. Or use built-in options at training time to perform weight balancing."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data engineering\n",
    "\n",
    "I have performed a very basic data engineering here:\n",
    "    * Numerical missing values are replaced by the mean of the columns\n",
    "    * Categorical missing values are replaced by the most common\n",
    "    * Numeric data is scaled\n",
    "    * Categorical data is one hot encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['id', 'target', 'ps_ind_01', 'ps_ind_02_cat', 'ps_ind_03',\n",
      "       'ps_ind_04_cat', 'ps_ind_05_cat', 'ps_ind_06_bin', 'ps_ind_07_bin',\n",
      "       'ps_ind_08_bin', 'ps_ind_09_bin', 'ps_ind_10_bin', 'ps_ind_11_bin',\n",
      "       'ps_ind_12_bin', 'ps_ind_13_bin', 'ps_ind_14', 'ps_ind_15',\n",
      "       'ps_ind_16_bin', 'ps_ind_17_bin', 'ps_ind_18_bin', 'ps_reg_01',\n",
      "       'ps_reg_02', 'ps_car_01_cat', 'ps_car_02_cat', 'ps_car_04_cat',\n",
      "       'ps_car_06_cat', 'ps_car_07_cat', 'ps_car_08_cat', 'ps_car_09_cat',\n",
      "       'ps_car_10_cat', 'ps_car_11_cat', 'ps_car_11', 'ps_car_12', 'ps_car_13',\n",
      "       'ps_car_14', 'ps_car_15', 'ps_calc_01', 'ps_calc_02', 'ps_calc_03',\n",
      "       'ps_calc_04', 'ps_calc_05', 'ps_calc_06', 'ps_calc_07', 'ps_calc_08',\n",
      "       'ps_calc_09', 'ps_calc_10', 'ps_calc_11', 'ps_calc_12', 'ps_calc_13',\n",
      "       'ps_calc_14', 'ps_calc_15_bin', 'ps_calc_16_bin', 'ps_calc_17_bin',\n",
      "       'ps_calc_18_bin', 'ps_calc_19_bin', 'ps_calc_20_bin'],\n",
      "      dtype='object')\n",
      "Encoding null values\n",
      "Processing categorical features...\n",
      "Number of missing values for feature: ps_ind_02_cat\n",
      "216\n",
      "Number of missing values for feature: ps_ind_04_cat\n",
      "83\n",
      "Number of missing values for feature: ps_ind_05_cat\n",
      "5809\n",
      "Number of missing values for feature: ps_car_01_cat\n",
      "107\n",
      "Number of missing values for feature: ps_car_02_cat\n",
      "5\n",
      "Number of missing values for feature: ps_car_04_cat\n",
      "0\n",
      "Number of missing values for feature: ps_car_06_cat\n",
      "0\n",
      "Number of missing values for feature: ps_car_07_cat\n",
      "11489\n",
      "Number of missing values for feature: ps_car_08_cat\n",
      "0\n",
      "Number of missing values for feature: ps_car_09_cat\n",
      "569\n",
      "Number of missing values for feature: ps_car_10_cat\n",
      "0\n",
      "Number of missing values for feature: ps_car_11_cat\n",
      "0\n",
      "Processing numerical features...\n",
      "Number of missing values for feature: id\n",
      "0\n",
      "Number of missing values for feature: target\n",
      "0\n",
      "Number of missing values for feature: ps_ind_01\n",
      "0\n",
      "Number of missing values for feature: ps_ind_03\n",
      "0\n",
      "Number of missing values for feature: ps_ind_14\n",
      "0\n",
      "Number of missing values for feature: ps_ind_15\n",
      "0\n",
      "Number of missing values for feature: ps_reg_01\n",
      "0\n",
      "Number of missing values for feature: ps_reg_02\n",
      "0\n",
      "Number of missing values for feature: ps_car_11\n",
      "5\n",
      "Number of missing values for feature: ps_car_12\n",
      "1\n",
      "Number of missing values for feature: ps_car_13\n",
      "0\n",
      "Number of missing values for feature: ps_car_14\n",
      "42620\n",
      "Number of missing values for feature: ps_car_15\n",
      "0\n",
      "Number of missing values for feature: ps_calc_01\n",
      "0\n",
      "Number of missing values for feature: ps_calc_02\n",
      "0\n",
      "Number of missing values for feature: ps_calc_03\n",
      "0\n",
      "Number of missing values for feature: ps_calc_04\n",
      "0\n",
      "Number of missing values for feature: ps_calc_05\n",
      "0\n",
      "Number of missing values for feature: ps_calc_06\n",
      "0\n",
      "Number of missing values for feature: ps_calc_07\n",
      "0\n",
      "Number of missing values for feature: ps_calc_08\n",
      "0\n",
      "Number of missing values for feature: ps_calc_09\n",
      "0\n",
      "Number of missing values for feature: ps_calc_10\n",
      "0\n",
      "Number of missing values for feature: ps_calc_11\n",
      "0\n",
      "Number of missing values for feature: ps_calc_12\n",
      "0\n",
      "Number of missing values for feature: ps_calc_13\n",
      "0\n",
      "Number of missing values for feature: ps_calc_14\n",
      "0\n",
      "Binarizing...\n",
      "Scaling...\n",
      "Finished !\n"
     ]
    }
   ],
   "source": [
    "data = encoding().Binarize_scale(dat=data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['id', 'ps_ind_01', 'ps_ind_02_cat', 'ps_ind_03', 'ps_ind_04_cat',\n",
      "       'ps_ind_05_cat', 'ps_ind_06_bin', 'ps_ind_07_bin', 'ps_ind_08_bin',\n",
      "       'ps_ind_09_bin', 'ps_ind_10_bin', 'ps_ind_11_bin', 'ps_ind_12_bin',\n",
      "       'ps_ind_13_bin', 'ps_ind_14', 'ps_ind_15', 'ps_ind_16_bin',\n",
      "       'ps_ind_17_bin', 'ps_ind_18_bin', 'ps_reg_01', 'ps_reg_02',\n",
      "       'ps_car_01_cat', 'ps_car_02_cat', 'ps_car_04_cat', 'ps_car_06_cat',\n",
      "       'ps_car_07_cat', 'ps_car_08_cat', 'ps_car_09_cat', 'ps_car_10_cat',\n",
      "       'ps_car_11_cat', 'ps_car_11', 'ps_car_12', 'ps_car_13', 'ps_car_14',\n",
      "       'ps_car_15', 'ps_calc_01', 'ps_calc_02', 'ps_calc_03', 'ps_calc_04',\n",
      "       'ps_calc_05', 'ps_calc_06', 'ps_calc_07', 'ps_calc_08', 'ps_calc_09',\n",
      "       'ps_calc_10', 'ps_calc_11', 'ps_calc_12', 'ps_calc_13', 'ps_calc_14',\n",
      "       'ps_calc_15_bin', 'ps_calc_16_bin', 'ps_calc_17_bin', 'ps_calc_18_bin',\n",
      "       'ps_calc_19_bin', 'ps_calc_20_bin'],\n",
      "      dtype='object')\n",
      "Encoding null values\n",
      "Processing categorical features...\n",
      "Number of missing values for feature: ps_ind_02_cat\n",
      "307\n",
      "Number of missing values for feature: ps_ind_04_cat\n",
      "145\n",
      "Number of missing values for feature: ps_ind_05_cat\n",
      "8710\n",
      "Number of missing values for feature: ps_car_01_cat\n",
      "160\n",
      "Number of missing values for feature: ps_car_02_cat\n",
      "5\n",
      "Number of missing values for feature: ps_car_04_cat\n",
      "0\n",
      "Number of missing values for feature: ps_car_06_cat\n",
      "0\n",
      "Number of missing values for feature: ps_car_07_cat\n",
      "17331\n",
      "Number of missing values for feature: ps_car_08_cat\n",
      "0\n",
      "Number of missing values for feature: ps_car_09_cat\n",
      "877\n",
      "Number of missing values for feature: ps_car_10_cat\n",
      "0\n",
      "Number of missing values for feature: ps_car_11_cat\n",
      "0\n",
      "Processing numerical features...\n",
      "Number of missing values for feature: id\n",
      "0\n",
      "Number of missing values for feature: ps_ind_01\n",
      "0\n",
      "Number of missing values for feature: ps_ind_03\n",
      "0\n",
      "Number of missing values for feature: ps_ind_14\n",
      "0\n",
      "Number of missing values for feature: ps_ind_15\n",
      "0\n",
      "Number of missing values for feature: ps_reg_01\n",
      "0\n",
      "Number of missing values for feature: ps_reg_02\n",
      "0\n",
      "Number of missing values for feature: ps_car_11\n",
      "1\n",
      "Number of missing values for feature: ps_car_12\n",
      "0\n",
      "Number of missing values for feature: ps_car_13\n",
      "0\n",
      "Number of missing values for feature: ps_car_14\n",
      "63805\n",
      "Number of missing values for feature: ps_car_15\n",
      "0\n",
      "Number of missing values for feature: ps_calc_01\n",
      "0\n",
      "Number of missing values for feature: ps_calc_02\n",
      "0\n",
      "Number of missing values for feature: ps_calc_03\n",
      "0\n",
      "Number of missing values for feature: ps_calc_04\n",
      "0\n",
      "Number of missing values for feature: ps_calc_05\n",
      "0\n",
      "Number of missing values for feature: ps_calc_06\n",
      "0\n",
      "Number of missing values for feature: ps_calc_07\n",
      "0\n",
      "Number of missing values for feature: ps_calc_08\n",
      "0\n",
      "Number of missing values for feature: ps_calc_09\n",
      "0\n",
      "Number of missing values for feature: ps_calc_10\n",
      "0\n",
      "Number of missing values for feature: ps_calc_11\n",
      "0\n",
      "Number of missing values for feature: ps_calc_12\n",
      "0\n",
      "Number of missing values for feature: ps_calc_13\n",
      "0\n",
      "Number of missing values for feature: ps_calc_14\n",
      "0\n",
      "Binarizing...\n",
      "Scaling...\n",
      "Finished !\n"
     ]
    }
   ],
   "source": [
    "data_valid = encoding().Binarize_scale(dat=data_valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Features\n",
    "\n",
    "Since the categorical features have one-hot encoded, I'll remove the original columns for the data. Then, create a train/test split using these features.\n",
    "\n",
    "The train/test split is done before dealing with data imbalance so that the evaluation on the test set is not biased"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 214 features columns\n"
     ]
    }
   ],
   "source": [
    "feat = list(data.columns)\n",
    "for f in feat:\n",
    "    if (f.split('_')[-1]) == 'cat':\n",
    "        feat.remove(str(f))\n",
    "features = np.array(feat)\n",
    "target = \"target\"\n",
    "# Remove first column and target\n",
    "features = np.delete(features,[0,1])\n",
    "train, test = train_test_split(data, train_size=0.80, test_size=0.20)\n",
    "features = list(features)\n",
    "print(\"There are {0} features columns\".format(len(features)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fine. Data is ready, we can now proceed with some model building."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple logistic regression model\n",
    "\n",
    "Logistic regression is one of the simplest technique we can use for binary classification.\n",
    "\n",
    "Note that data imbalance is dealt with via LogisticRegression parameter 'class_weight'. \"0\" are slightly underbalanced while \"1\" are largely overbalanced.\n",
    "\n",
    "To evaluate the model performance we'll use auc_roc scoring."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total fitting time: 77.11302518844604s\n",
      "roc_auc score: 0.626349975379408\n",
      "Total positive values on test set: 4291.0\n",
      "Total positive predictions on the test set via LogisticRegression: 16458.0\n",
      "There are 13.83% of positive values on the prediction test\n",
      "This model predicts 13.88% of positive results on validation set\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xd4VNXTwPHvkFCk9957ExAiCEpT\npFmQ14YdG4Jib9gR0Z9gQUUQaYpYwC5IEyuItFClE0KvAUInkDLvH2cTY0zIErItmc/z7JO7d+/u\nnWtwJ+eec+aIqmKMMcYA5Al0AMYYY4KHJQVjjDEpLCkYY4xJYUnBGGNMCksKxhhjUlhSMMYYk8KS\ngjHGmBSWFIwxxqSwpGCMMSZFeKADOFulS5fW6tWrBzoMY4wJKUuWLNmvqmUyOy7kkkL16tWJjIwM\ndBjGGBNSRGSrN8fZ7SNjjDEpLCkYY4xJYUnBGGNMCksKxhhjUlhSMMYYk8JnSUFExovIPhFZlcHr\nIiLviUiUiKwUkea+isUYY4x3fNlS+BjoeobXuwF1PI8+wAc+jMUYY4wXfDZPQVXniEj1MxzSA/hE\n3XqgC0SkuIhUUNXdvorJGGNCzeGT8cxevIkj23YR0aE5TSoX9+n5Ajl5rRKwPdXzHZ59/0kKItIH\n15qgatWqfgnOGGP8KT4xiXW7j7Jq12E27j3Gws0HiE9MotSiebw+czhH8xdi2bezc3RSkHT2aXoH\nqupoYDRAREREuscYY0woOXk6kakrdrHt4Ak+mb+FI3EJ/3q9foFEHpoxmisXTeN41RqUHTuaxm1q\n+DyuQCaFHUCVVM8rA7sCFIsxxvhczNFTvPXTerYcOM6C6IMp+6uWLEjdckW4tEFZmlYuTrOKRSgU\ncQGsXw9PPUWhgQPhvPP8EmMgk8IUoL+ITAJaAYetP8EYk5MkJinT/97Nql2H+XbpTmKOnkp5rUuj\ncjSrUoKbW1alWMG8bueBA1CyJIjAq69ClSoQEeHXmH2WFETkC6ADUFpEdgAvAXkBVHUUMB3oDkQB\nJ4A7fRWLMcb4mqoSuTWWpVtjOX4qgV/W7WP1riMpr1crVZA2tUpxT9saXFq/XNo3w2efwcMPw+uv\nw733Qs+efr4Cx5ejj27K5HUFHvDV+Y0xxtcSEpNYvCWWzxdtY+WOQ2w9cCLltRIF89K6ZikurV+W\n/2teiVKF86f/Idu3Q9++MH06XHQRXHyxn6JPX8iVzjbGmECKT0zimyU7mLFqD39siEnZX6pQPh6+\nrA6XNShLgwpFyRvmxTSwL76A++6DxER45x3o3x/CwnwYfeYsKRhjTCZijp5iyopdfLZwK9Exx1P2\nN61cjCuaVOCmllUpUiDv2X9wiRLQqhWMHg01fD+yyBuWFIwxJg1V5dd1+/h13T6mrtj1r+GidcoW\npmfzSvRuU52C+c7yKzQhAYYNg9On4bnnoGtX6NLFdSwHCUsKxhjjcejEaYbOWs/kxdtJTPpnStQF\nVYvzZJd6tKpRirA8WfwCX7EC7r4bliyBG25wncsiQZUQwJKCMSaXS0pS/tgQw9BZ61m7240WyheW\nh0c716Fn88pUKn6O8wNOnYLBg92oopIl4auv4Nprgy4ZJLOkYIzJldbtOcKw2RuYtXovAPnC89C0\ncjH6tKvFFU0qZN+JNm6EIUPg5pvh7behVKns+2wfsKRgjMk1VJXPF21jyIx1Kf0E5Yrm586La3B7\n62pn30eQkWPH4Icf4JZboHFjWLcOatbMns/2MUsKxpgcLSlJGTVnE+P/3ML+Y//MKL6ySQXua1eL\n8ysXy94Tzp4NffrA1q3QvDk0aBAyCQEsKRhjcpikJGXB5gNMXbGbH1fs4uipf0YONahQlKubVuTW\ni7I4hPRMYmPhiSdg/HioWxf++MMlhBBjScEYkyOs3nWY+ZsO8O7PG1MSQaF8YVxQtThXnF+Buy+p\ngfiqczcx0c1E3rABnnkGXnwRChTwzbl8zJKCMSZkbdx7lMmLtzNj1R52HjoJuA7jAd3q06VReWqU\nLuTbAPbvdyOKwsLgtdegalV3yyiEWVIwxoSUpCTltelrmbpyF3uPuD6C0oXz82SXerSsUZKmlYuT\nL9yXKw3j5hhMnAiPPOKGmvbpA9dc49tz+oklBWNMSNhzOI6P5m1m2t+72RHrWgU3tazKLa2q0rhS\nNncWn8nWra5e0axZ0KYNtGvnv3P7gSUFY0xQi4tPZPy8zbz/axQnTidSMF8YT3WtR7/2tXzXR5CR\nTz+Ffv1cS2H4cLj/fsjj41aJn1lSMMYEHVVlR+xJvl6yg4kLtnLw+GkAvryvNS1rlAxcYGXKuA7l\nDz+EatUCF4cPWVIwxgSNX9ft5d2fN7Jix+GUfQXy5mHEzc3p3Kicd+Wos1N8PLz1lvv5wguueF3n\nzkFboiI7WFIwxgSUqjLityg+mb+VfZ7lKhtXKkqbWqVpWrk4XRqVI9zfyQBg2TJXwG7ZMujVK2gL\n2GU3SwrGmICIT0xiwl9b+OD3TRw4fprShfNx60VVua9dLaqULBi4wOLiYNAgGDoUSpeGb76B//u/\nwMXjZ5YUjDF+N2dDDLePX5TyfMi153N9iyrkyWpZ6uwUFQVvvgm33+5uHZUoEeiI/MqSgjHGb1Zs\nP8TAqatZtu0QAM9f0YBbL6pGgbyBXYKSY8fgu+/gtttcAbv164NmJTR/s6RgjPEpVeX3DTEM/2Uj\nSz3JoFODsrzW83zKFg2CUhCzZrnJZ9u3Q0SEq1eUSxMCWFIwxvjIlv3H+X39Psb+uTllslmnBmV5\n4cqGVCvl4/IT3jhwAB57DD75BOrXh7lzQ7KAXXazpGCMyVYxR0/x8tTV/Lhyd8q++9rX5IGOtSma\n3ZVJsyq5gF1UlFsr+fnnQ7aAXXazpGCMyRZ/bIjhvV82smRrLABtapWiX4daXFi9ZOD7DJLFxLiV\nz8LC3Gpo1apBs2aBjiqoWFIwxmSZqhK5NZbnv1vF+r1HAWhZoyT3XFKDzo3KBzi6VFTh44/d7aLX\nX3e1i3r0CHRUQcmSgjEmS6au2MWDXywDIDyP8H/NK/HSlY0oVjBIbhEl27LFdSTPng1t20LHjoGO\nKKhZUjDGnJVl22IZ++dmpnn6DK5tXpmnu9WjbJEgvCc/caIrYCcCI0e6FkIOK2CX3SwpGGO8cjQu\nnr6fLmFe1AEAujQqx9DrmlLsvCBrGaRWrpwrbT1qlFsAx2TKkoIxJlPbD57gvolLWLP7CBfXLsXL\nVzeidtkigQ7rv+LjXXmKxES3JGbnzu5hvGZJwRiTofV7jtL9vbkkJingZiDf07ZmgKPKwNKlcNdd\nsGIF3HzzPwXszFnxaVIQka7Au0AYMFZVX0/zelVgAlDcc8wAVZ3uy5iMMWemqny6cBvfLt2RUo4i\noloJnulenxbVAriWQUZOnoSXX3b1isqUceUqcsjSmIHgs6QgImHACOByYAewWESmqOqaVIc9D3yp\nqh+ISENgOlDdVzEZYzIWte8YH/+1mZ9W700pYV2/fBHeuqEpjSr6cbnLsxUdDW+/Db17wxtv5LoC\ndtnNly2FlkCUqkYDiMgkoAeQOikoUNSzXQzY5cN4jDFpnEpI5KvIHQz/dSN7j7hEUKRAOK9c05ib\nLqwSmHUMvHHkCHz7rUsEjRrBxo05diU0f/NlUqgEbE/1fAfQKs0xA4GfRORBoBDQyYfxGGNSWbzl\nINePmg9A8YJ56d2mOrdeVDU4O5BTmz4d+vaFnTuhVStXr8gSQrbxZVJIr4dH0zy/CfhYVd8SkdbA\nRBFprKpJ//ogkT5AH4CqNqzMmHMyd2MMQ2auY9XOI5QunJ++7Wty9yU1kGDvlN2/Hx59FD79FBo2\nhHnzrICdD/gyKewAqqR6Xpn/3h66G+gKoKrzRaQAUBrYl/ogVR0NjAaIiIhIm1iMMV6IjjnGPZ9E\nEh1zHHCTzp7sUo/yxYJw0llayQXsoqPdUNNnn4X8+QMdVY7ky6SwGKgjIjWAnUAv4OY0x2wDLgM+\nFpEGQAEgxocxGZPr7Dkcx3Pf/c0v69zfWjdGVOGxznUpFwxrGWRm7143oigszI0uqlYNmjQJdFQ5\nms+SgqomiEh/YBZuuOl4VV0tIoOASFWdAjwOjBGRR3G3lnqrqrUEjMkGq3cdZvgvUcxcvQeAy+qX\n5Znu9YO/zwDcHIPx4+Hxx10Bu7594aqrAh1VruDTeQqeOQfT0+x7MdX2GuBiX8ZgTG6zKeYYb8xc\nn5IM2tYpzaOX16V51RAZqhkdDffeC7/+Cu3bQycbf+JPNqPZmBziaFw8L3y/iu+Xu667ltVLMrhn\nY+qWC4GWQbIJE+D++93tolGjXHKwAnZ+ZUnBmBxg7Nxo3v15I0dPJdCpQTke71yXBhWKZv7GYFOx\nIlx6KXzwAVSuHOhociVLCsaEqKQk5ce/d/O/6WvZfTiOEgXz8vk9rWhTu3SgQ/Pe6dOuzyApCQYO\nhMsvdw8TMJYUjAlBcfGJ9Jm4hDkbYihZKB+3t67Gi1c2DN4ZyOlZvNgVsFu1Cm67zQrYBQlLCsaE\nkLj4REbPieaLRdvYfTiOm1pWZVCPRuQNpWRw4oSbazBsGFSoAFOm2MiiIGJJwZgQoKp88Mcmhs5c\nD0C+sDy8c2MzrrmgUoAjy4LNm2H4cNeJPGQIFAviYnu5kCUFY4Lc5v3H6fNJJBv3HQNgUI9G3HZR\nteAvS5Ha4cOugN2dd7oCdlFRUKVK5u8zfmdJwZggpap8tnAbg6au4XRiEte3qMzgno3JHx4W6NDO\nzrRpbm3k3buhdWuoX98SQhCzpGBMENqw9yj3TIhk28ET5A/Pww8PXEzTKsUDHdbZiYmBRx6Bzz+H\nxo1dS6F+/UBHZTJhScGYIPP7+n30/mgxAL3bVOe5KxqEVkcyuAJ2l1zi+g9efhkGDIB8+QIdlfGC\nJQVjgsSuQye5dexCove7KqZT+l9Mk8oh1jrYswfKlnUzkt96C6pXd60EEzJC7M8PY3Ke+MQk3p69\ngbZDfyN6/3FaVi/JgmcuC62EkJQEH34Ideu6nwBXXmkJIQRl2lIQkfOAR4BqqtpXRGoDdVR1hs+j\nMyYHO3YqgXdmb2Digq2cSkiiUL4wJt/XmsaVQmyIZlSUG176+++uREWXLoGOyJwDb24fjQf+Bi7x\nPN8FfAVYUjAmC5KSlHF/buaNn9ZzOiGJfOF5GHxNY3oF85rIGfnoI1fALl8+GDMG7r7bZiWHOG+S\nQh1VvUlErgdQ1RMSUgOkjQkOcfGJfDJ/C18s2s7m/ccpdl5eXrqqIde3qEK+8BBLBsmqVnUtgxEj\noFIITqQz/+FNUjjtWSZTATwrqZ32aVTG5CCqyusz1vHdsp3sO3qKmqUL8UqPRtzSqhp58oTY31en\nTsH//uf6EAYNgssucw+TY3iTFF4BZgKVRWQC0B64x6dRGZNDLN0Wyxsz1zM/+gB5BN676QKualIh\ntGYjJ1u40N0eWr0a7rjDCtjlUJkmBVWdISKRQBtAgCdVdZ/PIzMmhG07cIJXpq1h9pq9ADzRuS59\n29cKvT4DgOPH4YUX4J133C2iH3+EK64IdFTGR7wZffSTqnYGfkhnnzEmjWkrd/PA50sBuKZZRR7v\nXI8qJQsGOKpzsHUrjBzp1kl+/XUoGoKL9xivZZgURCQfUAAoJyJFcK0EgKJAVT/EZkxIiTl6ihd/\nWMWMVXsIzyN8ek8rLqpZKtBhZc2hQ/D113DPPdCwoRt2aiuh5Qpnaik8ADwGlAVW809SOAKM8nFc\nxoSM6JhjPPblCpZvPwTAVU0r8mrPxhQtkDfAkWXRDz9Av36wb58rVVG/viWEXCTDpKCqw4BhIvKI\nqr7jx5iMCRkz/t5Nv8/craL65YvwSKe6dG1cPsBRZdG+ffDQQzB5MjRp4ha/sQJ2uY43Hc3viEh9\noCHudlLy/s99GZgxwe6ryO08+fVKAEbc3JwrmlQIcETnIDERLr4Ytm2DwYPhqacgb4i2dMw58aaj\n+XmgM1AfmAV0Af4ELCmYXCk+MYmRv21i2M8bKF4wLz890o6yRQtk/sZgtGsXlC/vCti9+64rYNew\nYaCjMgHkzfi4G4GOwG5VvQ1oilVXNbnUhr1H6fjm7wz7eQPnVyrGrFBNCElJ8MEH7vbQKE8XYffu\nlhCMV1/uJ1U1UUQSPKOQ9gA1fRyXMUFl1c7DvDptLfOjDwDw/BUNuPuSGqE5CW3DBlfAbs4c6NQJ\nunULdEQmiHiTFJaJSHFcYbxI3OijpT6NypggsXRbLE9+tYJNMW6Ng26Ny/Po5XWpW65IgCPLonHj\noH9/KFAAxo+H3r1tVrL5lzMmBU/hu4GqeggYISKzgKKqaknB5GjxiUkMnbmOMXM3A3BZ/bK83KMR\nlUuE8CQ0cH0G3bq5AnYVQrhj3PjMGZOCqqqI/Ai08DyP8ktUxgSIqvLz2n28MWsdG/Yeo3PDcrxw\nZcPQnZF86hS88orbHjzYCtiZTHlz+2iRiDS31oHJyVSVORv3c8f4RSn7BvVoxG0XVQvNfgOAv/5y\nBezWrYO77rICdsYr3iSFS4B7RWQTcBw3s1lVtblPIzPGD47GxfP27A18vWQHR+MSKFIgnEvrl+Xl\nqxtRvGCILjR/7Bg89xwMHw5VqsDMmbYamvGaN0nhmqx+uIh0Bd4FwoCxqvp6OsfcAAzErdewQlVv\nzur5jDkbH83bzNs/beDoqQTKFy3AbRdVo0+7mqGbDJJt2+bWSX7gAXjtNSgSop3iJiC8mdG8KSsf\nLCJhwAjgcmAHsFhEpqjqmlTH1AGeAS5W1VgRKZuVcxlzNo6fSqDPxEjmRR2gQN48TLy7JW3rlAl0\nWOcmNha++gr69HFzDaKjoWLFQEdlQpAvJ6G1BKJUNRpARCYBPYA1qY65FxihqrEAtk6D8aW4+EQ+\nmreFkb9HcTQugUtql2bkrc1Dt3Bdsu++c+skx8RA+/ZQr54lBJNlvkwKlYDtqZ7vAFqlOaYugIjM\nw91iGqiqM9N+kIj0AfoAVK1qVbvN2Tl5OpGxc6OZHLmdHbEnaVypKAO6NuCSOqUDHdq52bMHHnzQ\nlbhu1gymTXMJwZhz4FVSEJHKQB1V/U1E8gPhqno8s7els0/TOX8doANQGZgrIo098yL+eZPqaGA0\nQERERNrPMCZDcfGJtHvjN2KOngJgyLXnc+OFOeAPi8REaNsWtm93/QZPPGEF7Ey28KYg3l1Af6AY\nUAuoBowEOmXy1h1AlVTPKwO70jlmgarGA5tFZD0uSSz2KnpjzmDP4ThuH7+QmKOn6N+xNo9eXpew\nPCE+JHPHDndrKCwM3nsPatSw8tYmW3lTEO8h4CJceQtUdQNu4Z3MLAbqiEgNzypuvYApaY75Hlds\nDxEpjbudFO1d6Mak7+Dx0/T7dAkX/e8XNuw9Ru821XmiS73QTghJSW6Iaf36rpAduJnJlhBMNvPm\n9lGcqp5OnsDjGVWU6f9dqpogIv1x5bbDgPGqulpEBgGRqjrF81pnEVkDJAJPquqBLF6LyeUSk5Tn\nvvubSYtdV1bpwvl476YLaFMrxPsO1q1zy2LOm+fmG1x5ZaAjMjmYN0lhnog8BRQQkY64ZTp/9ObD\nVXU6MD3NvhdTbStuyc/HvI7YmDQSEpMYPTeaoTPXp+z74JbmdDs/B9T2GTvWFbArWBAmTIDbbrNZ\nycanvEkKT+FG/qwDHsb9df+hL4Myxlub9x/nya9WELk1FoB7LqnB81fmoDUBatWCq66C99+HcuUC\nHY3JBbxJCt1xs5E/8HUwxngrLj6RGat28+jkFQDcfUkNBnSrT94wb7rJglhcHAwa5LZfew06dnQP\nY/zEm6RwA/C+iPwKTAJ+VtVE34ZlTMa+itzOc9+t4nRiEkULhPPWDc24vGEO+Ct63jxXwG79eteH\nYAXsTAB4U+biNs/chCuAu4DRIjJDVfv6PDpjUklMUh6atIxpK3dTunB+Hr28DjdEVAn91sHRo/Ds\ns26Ng2rVYNYs6Nw50FGZXMqryWuqekpEfgBO4kYS3QBYUjB+s2xbLH0/XcLeI6doVqU4k/pcRIG8\nYYEOK3vs2OE6lB98EF59FQoXDnREJhfzZvJaJ9wcg07APOATwCqZGr84diqBq9//k2jPcpjPX9GA\ne9rmgCXCDxyAL7+Efv2gQQNXwM5WQjNBwJuWQl9cX8KDqnrSx/EYkyJq31Ee/3IF0THH6dSgLIN6\nNKZi8fMCHda5UYVvvnFlrQ8ehEsvdfWKLCGYIOFNn8J1/gjEmGSJScrT36zk6yU7ABh4VUN6X1wj\nwFFlg927XTL47jto0QJ++skK2Jmgk2FSEJE/VLW9iMTy70J2ySuvlfR5dCbXWbzlIA9/sYxdh+Oo\nWKwAY+6IoFHFYoEO69wlF7DbuROGDoVHH4VwXxYpNiZrzvSvMnlwdIjXCDChIDFJ+X7ZTh7/ys07\neLJLPe7vUCt010dOtn07VKrkCtiNGOEK2NWtG+iojMlQhmP5VDXJszlOVRNTP4Bx/gnP5AbLtx+i\n1rPTefyrFRTOH86U/hfzQMfaoZ0QEhNdFdPUBey6dLGEYIKeN+3XJqmfeAriXeibcExu8/2ynTwy\neTkAVzWtyBvXNQn9oaZr17pJaPPnu0qmV10V6IiM8dqZ+hSeBgYARUTkYPJuXP+CtRTMOYmLT+SJ\nr1bw48rdFC+Yl0l9LqJ++aKBDuvcjR7t5hsUKQITJ8Itt9isZBNSztRSGAq8BfwPlxwAsBIX5lyt\n2nmY60fN52R8Ii1rlGTsHRGhv05ysjp1oGdPd+uorDfLjhgTXM6UFGqr6kYRmQg0St6ZfJ9XVVf6\nODaTA42dG83gaWvJGyY8060+97WvFeiQzs3JkzBwoGsNvP66FbAzIe9MSWEAcDcwIp3XFGjnk4hM\njvXrur0MnrYWgJmPtKNWmRAv5zBnjitct3Ej9O1rBexMjpBhUlDVuz0/2/ovHJNT/bB8Jw9PWk7p\nwvmZ/Wg7ShTKF+iQsu7IERgwwI0qqlkTfvnFzUw2JgfItLykiPyfiBTxbA8QkS9FpKnvQzM5wfFT\nCTzw+VIenrScAnnzMO6OiNBOCAC7dsHHH8Njj8HKlZYQTI7izZDUgar6rYi0Aa4C3satvHaRTyMz\nIU1V+XTBVl74YTUATSsX4+M7W4ZuQti/3xWwu/9+N/dg82ZbCc3kSN4kheTRRlcCI1X1GxF53ocx\nmRA3d2MMT3+9kl2H4wB4t1czejSrFOCoskjVJYMHH4RDh6BTJzcBzRKCyaG8SQq7RWQE0A1oISL5\n8OK2k8l94uITeezL5Uz/ew9heYQnOtflnrY1Q3cy2q5drrT1lCkQEeH6DmxGssnhvF2OszswXFVj\nRaQiqeYtGAMQn5jEg18sY/aavdwYUYVnuteneMEQvVUErkxFu3augN2bb8LDD1sBO5MreFM6+5iI\nrAE6iEgHYK6qzvB5ZCZkHDx+mlvGLmTt7iP071ibJ7qEcDnorVuhcmVXwG7kSDe6qHbtQEdljN94\nM/qoP/AlUNXz+FJE7vd1YCY0LNkaS8c3f2ft7iM8171B6CaExER4+223ClpyAbvOnS0hmFzHm/Zw\nH6Clqh4DEJHXgL+Akb4MzAS32OOnGfDtSmat3ksegVG3tqBr4/KBDitrVq1yBewWLYIrr4Rrrgl0\nRMYEjDdJQYD4VM/jPftMLjXj7930+2wpAI0rFeWDW1pQpWTBAEeVRaNGwUMPQbFi8Pnn0KuXzUo2\nuZo3SWEisEBEvsElg2uACT6NygSlpCTlsS+X8/3yXRQtEM6Qa5vQ7fwQXVs4uSRFgwZw/fXwzjtQ\npkygozIm4ERVMz9I5EIgudzFXFVd7NOoziAiIkIjIyMDdfpc6+Dx07w0ZTVTV+yiXd0yDO91AcUK\nhmBl0xMn4MUXXUfykCGBjsYYvxGRJaoakdlx3o6xO+V5JHl+mlzkr037uXnMQgB6NKvIOzc2C81V\n0X7/3RWw27TJzUy2AnbG/Ic3o4+eA74AKgCVgc9F5BlfB2aCw/BfNnLzmIUUzh/O8Jsu4N1eF4Re\nQjh8GO6775+S1r/+6tZLDrXrMMYPvJmZfCtwoao+r6rPAS2B2735cBHpKiLrRSRKRDKc8CYi14mI\nikimTRvjHwePn+b57//mrdkbqF22MD8/1p6rmlYMdFhZs3s3fPopPPGEK2Bn6x0YkyFvbh9tTXNc\nOBCd2Zs8azmPAC4HdgCLRWSKqq5Jc1wR4CFgobdBG99asjWWm8cs4FRCEu3rlmH07S3IHx5ipSpi\nYmDSJFezqH592LLFOpKN8YI3SeEEsFpEZuEW1+kM/CkibwOo6mMZvK8lEKWq0QAiMgnoAaxJc9wr\nuKU/nzj78E12UlXe/Gk9I37bBMCY2yO4vGGIFX5ThS++cMNMjxyBLl1cvSJLCMZ4xZukMM3zSLbA\ny8+uBGxP9XwH0Cr1ASJyAVBFVX8UEUsKARQXn8h9E5fwx4YYmlYpzvs3XRB6cw+2b3cF7KZNg1at\nYNw4K2BnzFnypvbRuCx+dnq9eCnjX0UkDzAM6J3pB4n0wc2spmrVqlkMx2Tk9/X7GPDN3+w5Esc9\nl9Tg2e4NyJMnxDphExKgQwfYsweGDXO3jcJC7JaXMUHAl2UfdwBVUj2vDOxK9bwI0Bj43TOapTww\nRUSuVtV/TURQ1dHAaHDzFHwYc66SkJjEy1PXMHHBVgBe63k+N7cKsaS7ZQtUqeIqmH74oStgV7Nm\noKMyJmT5cl2ExUAdEanhWYOhFzAl+UVVPayqpVW1uqpWx92W+k9CML6x/eAJrhk5j4kLtnJx7VIs\nf/Hy0EoICQmupHWDBq6aKbgFcCwhGHNOvG4piEh+VfV64pqqJngqrM4CwoDxqrpaRAYBkao65cyf\nYHxl35E4+n++lFU7j/BIpzo80inE7ruvXOkK2EVGQo8ecO21gY7ImBwj06QgIi2BcUAxoKqINAXu\nUdUHM3uvqk4HpqfZ92IGx3bwJmCTdacTkhg6cx1j/9wMwNBrm3DDhVUyeVeQGTnSLXhTogRMnuzq\nFtkkNGOyjTcthfdw6zN/D6CqK0TEZv+EmNjjp7n+w/lE7TsGwEd3XkjHemUDHNVZSC5J0bixq2Q6\nbBiULh3oqIzJcbxJCnlUdWs9/xw0AAAWqUlEQVSa0gaJPorH+MDa3Ufo9u5cAJ7qWo++7WqFzuii\n48fh+eddR/Ibb7glMtu1C3RUxuRY3nQ0b/fcQlIRCRORR4ANPo7LZIPEJGXcn5vp9u5cCucPZ9St\nLbi/Q+3QSQi//ALnn+/KWp865VoLxhif8qal0A93C6kqsBf42bPPBLHFWw5y/2dLiTl6ipqlCzH0\nuiZEVC8Z6LC8c+iQq1M0bhzUqQNz5kDbtpm/zxhzzryZvLYPN5zUhICkJOWBz5cyY9UeAB68tDaP\ndqobOq0DgL17Xd2ip5+Gl16C884LdETG5BrejD4aQ6qZyMlUtY9PIjJZtudwHAOnrGbm6j1ULnEe\nE+5qSa0yhQMdlneSE8HDD0O9em5SmnUkG+N33tw++jnVdgGgJ/+uaWSCwNQVu3hk8nISk5SbWlbl\ntZ6NQ2PdA1X47DOXDI4dg+7d3S0jSwjGBIQ3t48mp34uIhOB2T6LyJy1/01fy4dzoqlU/DxG3NKc\nZlWKBzok72zbBn37wowZ0Lr1P30IxpiAyUrtoxpAtewOxJy92OOnufeTSCK3xlK/fBG+7NuaogVC\nZN3k5AJ2+/bBe++55TGtgJ0xAedNn0Is//Qp5AEOAhmuomb8Y9/ROG4YNZ8tB07Qp11NnupSj/Aw\nX5ayyibR0VCtmpt3MGYM1KoF1asHOipjjMcZv0XE3ZRuCpTxPEqoak1V/dIfwZn07Tx0kq7vzGXL\ngROMuLk5z3ZvEPwJISEBhgyBhg3d+sgAl11mCcGYIHPGbxJVVeA7VU30PGz2UIDtiD3BDaPmE3vi\nNO/2asYVTSoEOqTMLV/uFr0ZMMB1JF9/faAjMsZkwJs/LxeJSHOfR2IytWrnYS596w92HjrJu70u\noEezSoEOKXPvvw8XXgg7d8LXX8O330KFEEhkxuRSGfYpiEi4qiYAlwD3isgm4DhuRTVVVUsUfvTO\nzxt45+eN5BGYcFdL2tcN8jWHkwvYNWkCt9wCb78NJUNkRrUxudiZOpoXAc2Ba/wUi8nAb+v28c7P\nG6lasiBjbo+gXvkigQ4pY8eOwXPPQd68bhEcK2BnTEg5U1IQAFXd5KdYTDr+itpPv8+WULpwPqY/\n3JbC+X25guo5+ukn6NPHzT948MF/WgvGmJBxpm+YMiLyWEYvqurbPojHpPLHhhj6TlxC/vAwJvVp\nHbwJITYWHnsMPv7YlaiYMwcuuSTQURljsuBM3zJhQGE8LQbjX2PnRjN42lrKFMnPF/e2onbZIK5h\ntG+f60R+5hl48UUoUCDQERljsuhMSWG3qg7yWyQmxdCZ6xj5+yaaVinO+DsiKFU4f6BD+q89e+CL\nL+DRR/8pYFeqVKCjMsacozMNSbUWQgB8tnArI3/fRJPKxZjc56LgSwiqMGGCm4T2zDOwcaPbbwnB\nmBzhTEnhMr9FYVBVxs6N5rnvVtGiWgkm92lNgbxBVgtoyxbo2hV693ZJYflyK2BnTA6T4e0jVT3o\nz0Byuzd/Ws+I31wLYdwdEZyXL8gSQkICdOwI+/e7MhV9+0KeIC+tYYw5a0E6nCV3+XXdXkb8tom2\ndUoz7o4LyRceRF+2UVFQo4YrYDd+PNSs6QraGWNypCD69smdZvy9m7s+jqRUoXx8eFuL4EkI8fHw\n2mvQqNE/Bew6drSEYEwOZy2FAFqz6wj9PltKoXxhjLqtBQXzBcmvY+lSuPtu12dw/fVw442BjsgY\n4ydB8mdp7hMXn8h9n0ZSJH84Pz7UlgurB0ldoPfeg5Yt3ZDTb7+FL7+EcuUCHZUxxk8sKQTI0Jnr\n2X7wJG/e0JQapQsFOhw31BTgggvg9tthzRro2TOwMRlj/C5I7lfkHqrKgG/+ZnLkdi6rX5YujcoH\nNqCjR918g/z54a23oG1b9zDG5ErWUvCjwyfi6ffpUiZHbqd1zVK8f3OAq4/PnAmNG8PIka6lYGso\nGZPrWUvBT1SVvp8uYX70ATo1KMuY2yOQQFUQPXDAFbD75BNo0ADmzYPWrQMTizEmqFhLwU/e/Gk9\n86MP8NBldRh7x4WBSwjgksJ338ELL8CyZZYQjDEpfJoURKSriKwXkSgRGZDO64+JyBoRWSkiv4hI\njhwEP23lbkb8tolaZQrRv2PtwASxe7db9EYV6taFrVth0CDXl2CMMR4+SwoiEgaMALoBDYGbRKRh\nmsOWARGq2gT4Ghjqq3gC5Ze1e+n/xVJKFsrHN/3a+H9ymqqbidyggWsZREW5/SVK+DcOY0xI8OU3\nVEsgSlWjVfU0MAnokfoAVf1NVU94ni4AKvswHr/7ceUu+kxcQsVi5/HLY+0pXjCffwPYvBk6d3YT\n0Zo2hRUrrICdMeaMfNnRXAnYnur5DqDVGY6/G5iR3gsi0gfoA1C1atXsis+nRs/ZxGvT11GiYF4+\nvK0FJQr5OSEkJMCll7r+gw8+cMtkWgE7Y0wmfJkU0utJTXfMo4jcCkQA7dN7XVVHA6MBIiIign7c\n5Fs/rWf4r1HULVeYSX1aU9KfCWHjRle0LjwcPvoIatWCKlX8d35jTEjz5Z+OO4DU30aVgV1pDxKR\nTsBzwNWqesqH8fjFsNkbGP5rFO3qlmHaQ239lxDi42HwYDfv4P333b4OHSwhGGPOii9bCouBOiJS\nA9gJ9AJuTn2AiFwAfAh0VdV9PozFLzbuPcrwXzfSrEpxxt0RQd4wP92uiYx0/QYrV0KvXnDTTf45\nrzEmx/HZt5aqJgD9gVnAWuBLVV0tIoNE5GrPYW8AhYGvRGS5iEzxVTy+pqo8/c1KCuULZ9StLfyX\nEN59F1q1covf/PCDWze5bFn/nNsYk+P4dEazqk4HpqfZ92Kq7U6+PL8/PfvdKpZuO8STXepRvlgB\n359QFUQgIsK1EoYOheLFfX9eY0yOZmUussHSbbF8sWgb1zSryP0davn2ZEeOwNNPQ4ECMGwYXHyx\nexhjTDawMYrnaO+ROO76eDGF84fzdLf6vi1fMX26Wwlt9Gg3usgK2BljspklhXNw7FQCN41ewKET\n8Qy7sRkVip3nmxPt3w+33gpXXAHFisFff8Ebb7jbR8YYk40sKWTR6YQkbvxwPtH7j/PilQ25vKEP\nVyeLjYWpU+Gll9xSma3ONAfQGGOyzvoUsuB0QhI3j1nA6l1HeKprPe66pEb2n2TnTvjsM3jySVea\nYutW60g2xvictRSy4LXpa4ncGsvVTStyf4dsrnqqCmPGQMOGMHAgbNrk9ltCMMb4gSWFszThry18\n/NcWrmlWkfduuiB7P3zTJrjsMlenqHlzNxmtdoBKbRtjciW7fXQWNu8/zpCZ66hfvgivX9skez88\nIcElhIMH4cMP4Z57rICdMcbvLCl46VRCIvdMWAzAB7e2oEDesOz54PXrXdG68HCYMMFtV85RFcSN\nMSHE/hT10tNfr2RTjBtpVKN0oXP/wNOn4eWX4fzzYcQIt699e0sIxpiAspaCF16dtobvl+/izour\n06tlNqznsGiRK02xahXcfDPccsu5f6YxxmQDaylk4u3ZGxgzdzMNKxTl6a71z/0D33kHWrf+Z+7B\nZ59B6dLn/rnGGJMNLCmcwa5DJxn+60YuqlmS7x+4+Nz6EZJLUrRsCffeC6tXw5VXZk+gxhiTTez2\n0Rm8PXsDqjD4msbkC89i/jx8GJ56Cs47z7US2rRxD2OMCULWUsjA4i0H+XrJDrqfX57aZYtk7UOm\nTnWT0MaOhfz5rYCdMSboWVJIR+zx0zz99UoK5gtj8DXnn/0HxMS4DuSrr4ZSpWDBAhgyxArYGWOC\nniWFdAycupro/ccZel2TrK2xfPiwK3P98stuqcwLL8z+II0xxgesTyGNb5bs4Iflu+jdpjpXNqno\n/Ru3b4dPP4UBA1xpiq1bXZlrY4wJIdZSSGXF9kM8/c1KapYuxOOd63r3pqQkGDXKLX4zePA/Bews\nIRhjQpAlBY8Dx05x/2dLKXZeXj67txVFCuTN/E0bN8Kll0K/fm6o6d9/WwE7Y0xIs9tHuLpGnd7+\ng9gT8XzU+0LvVlBLSIDLL4dDh2DcOLjzTutINsaEPEsKwC1jFhJ7Ip6h1zahY/2yZz547Vq36E14\nOEyc6ArYVTyLvgdjjAliuf720Q/LdxK5NZbu55fnhgurZHzgqVNuOcwmTeD9992+tm0tIRhjcpRc\n3VKIT0zi9RnrqFisAG/f0CzjAxcscAXs1qyB225zD2OMyYFydUth/J+b2X04jgHdG2Rc1+itt1xZ\niqNH3dyDTz5xE9KMMSYHyrVJ4eDx07z/WxTnVyrGVU0q/PeApCT3s3Vr6NvXlbnu1s2/QRpjjJ/l\n2ttHD09axtG4BAZe3RBJPWro0CF4/HEoWBCGD7cCdsaYXCVXthSWbYtl7sb93HVxDVpUK/nPC99/\n7wrYTZgARYpYATtjTK6T65KCqvLiD6vJH56HBzrWcjv37YMbboCePaFcObcy2muv2bwDY0yuk+uS\nwryoA/y98zB929eiVOH8bueRIzB7Nrz6qksIzZsHNkhjjAkQnyYFEekqIutFJEpEBqTzen4Rmex5\nfaGIVPdlPElJynu/bCRfeB76VA93SUDVlabYtg2efRbyelHewhhjciifJQURCQNGAN2AhsBNItIw\nzWF3A7GqWhsYBgzxVTwAny/axuLN+xl/fBGFmjd1t4iSC9gVyeJCOsYYk4P4sqXQEohS1WhVPQ1M\nAnqkOaYHMMGz/TVwmYjvbuTP+uZ3fvjqeS55Z6Abarp6tRWwM8aYVHw5JLUSsD3V8x1Aq4yOUdUE\nETkMlAL2Z3cw3yzcwv9GP0UZjYOPPoI77rCOZGOMScOXLYX0vnHTjvH05hhEpI+IRIpIZExMTJaC\nqVymCJ/2H4ysXgO9e1tCMMaYdPiypbADSF1hrjKwK4NjdohIOFAMOJj2g1R1NDAaICIiIkuTB1rV\nLEWrV+/NyluNMSbX8GVLYTFQR0RqiEg+oBcwJc0xU4A7PNvXAb+q2owxY4wJFJ+1FDx9BP2BWUAY\nMF5VV4vIICBSVacA44CJIhKFayH08lU8xhhjMufT2keqOh2Ynmbfi6m244DrfRmDMcYY7+W6Gc3G\nGGMyZknBGGNMCksKxhhjUlhSMMYYk8KSgjHGmBQSatMCRCQG2JrFt5fGByU0gpxdc+5g15w7nMs1\nV1PVMpkdFHJJ4VyISKSqRgQ6Dn+ya84d7JpzB39cs90+MsYYk8KSgjHGmBS5LSmMDnQAAWDXnDvY\nNecOPr/mXNWnYIwx5sxyW0vBGGPMGeTIpCAiXUVkvYhEiciAdF7PLyKTPa8vFJHq/o8ye3lxzY+J\nyBoRWSkiv4hItUDEmZ0yu+ZUx10nIioiIT9SxZtrFpEbPL/r1SLyub9jzG5e/NuuKiK/icgyz7/v\n7oGIM7uIyHgR2SciqzJ4XUTkPc9/j5Ui0jxbA1DVHPXAleneBNQE8gErgIZpjrkfGOXZ7gVMDnTc\nfrjmjkBBz3a/3HDNnuOKAHOABUBEoOP2w++5DrAMKOF5XjbQcfvhmkcD/TzbDYEtgY77HK+5HdAc\nWJXB692BGbiVKy8CFmbn+XNiS6ElEKWq0ap6GpgE9EhzTA9ggmf7a+AykZBenzPTa1bV31T1hOfp\nAtxKeKHMm98zwCvAUCDOn8H5iDfXfC8wQlVjAVR1n59jzG7eXLMCRT3bxfjvCo8hRVXnkM4KlKn0\nAD5RZwFQXEQqZNf5c2JSqARsT/V8h2dfuseoagJwGCjll+h8w5trTu1u3F8aoSzTaxaRC4Aqqvqj\nPwPzIW9+z3WBuiIyT0QWiEhXv0XnG95c80DgVhHZgVu/5UH/hBYwZ/v/+1nx6SI7AZLeX/xph1h5\nc0wo8fp6RORWIAJo79OIfO+M1ywieYBhQG9/BeQH3vyew3G3kDrgWoNzRaSxqh7ycWy+4s013wR8\nrKpviUhr3GqOjVU1yffhBYRPv79yYkthB1Al1fPK/Lc5mXKMiITjmpxnaq4FO2+uGRHpBDwHXK2q\np/wUm69kds1FgMbA7yKyBXfvdUqIdzZ7+2/7B1WNV9XNwHpckghV3lzz3cCXAKo6HyiAqxGUU3n1\n/3tW5cSksBioIyI1RCQfriN5SppjpgB3eLavA35VTw9OiMr0mj23Uj7EJYRQv88MmVyzqh5W1dKq\nWl1Vq+P6Ua5W1cjAhJstvPm3/T1uUAEiUhp3Oynar1FmL2+ueRtwGYCINMAlhRi/RulfU4DbPaOQ\nLgIOq+ru7PrwHHf7SFUTRKQ/MAs3cmG8qq4WkUFApKpOAcbhmphRuBZCr8BFfO68vOY3gMLAV54+\n9W2qenXAgj5HXl5zjuLlNc8COovIGiAReFJVDwQu6nPj5TU/DowRkUdxt1F6h/IfeSLyBe72X2lP\nP8lLQF4AVR2F6zfpDkQBJ4A7s/X8IfzfzhhjTDbLibePjDHGZJElBWOMMSksKRhjjElhScEYY0wK\nSwrGGGNSWFIwQUtEEkVkeapH9TMcWz2jqpL+JiIRIvKeZ7uDiLRJ9VpfEbndj7E0C/Wqoca/ctw8\nBZOjnFTVZoEO4mx5JsglT5LrABwD/vK8Niq7zyci4Z4aXulphitrMj27z2tyJmspmJDiaRHMFZGl\nnkebdI5pJCKLPK2LlSJSx7P/1lT7PxSRsHTeu0VEhniOWyQitT37q4lbhyJ5PYqqnv3Xi8gqEVkh\nInM8+zqIyI+elk1f4FHPOduKyEAReUJEGojIojTXtdKz3UJE/hCRJSIyK70KmCLysYi8LSK/AUNE\npKWI/CVuTYG/RKSeZwbwIOBGz/lvFJFC4ur1L/Ycm15lWZObBbp2uD3skdEDNyN3uefxnWdfQaCA\nZ7sOblYrQHU89eeB4cAtnu18wHlAA2AqkNezfyRwezrn3AI859m+HfjRsz0VuMOzfRfwvWf7b6CS\nZ7u452eHVO8bCDyR6vNTnnuuq6Zn+2ngedzM1b+AMp79N+Jm8aaN82PgRyDM87woEO7Z7gR849nu\nDbyf6n2vAbcmxwtsAAoF+ndtj+B52O0jE8zSu32UF3hfRJrhkkbddN43H3hORCoD36rqRhG5DGgB\nLPaU+TgPyKgG1Bepfg7zbLcG/s+zPRG3RgPAPOBjEfkS+PZsLg5XxO0G4HXcl/+NQD1cIb/ZnjjD\ngIzq2nylqome7WLABE+rSPGURUhHZ+BqEXnC87wAUBVYe5axmxzKkoIJNY8Ce4GmuNuf/1k8R1U/\nF5GFwBXALBG5B1dueIKqPuPFOTSD7f8co6p9RaSV51zLPcnKW5Nxtai+dR+lG0XkfGC1qrb24v3H\nU22/Avymqj09t61+z+A9AlyrquvPIk6Ti1ifggk1xYDd6mrl34b7S/pfRKQmEK2q7+EqSjYBfgGu\nE5GynmNKSsbrVN+Y6ud8z/Zf/FM48RbgT8/n1FLVhar6IrCff5c0BjiKK+P9H6q6CdfaeQGXIMCV\nui4jbl0ARCSviDTKIM7UigE7Pdu9z3D+WcCD4mmGiKuea0wKSwom1IwE7hCRBbhbR8fTOeZGYJWI\nLAfq45YuXIO7Z/+Tp0N3NpDREob5PS2Nh3EtE4CHgDs9773N8xrAGyLyt2c47BzcGsKpTQV6Jnc0\np3OuycCt/LMewGlcOfchIrIC1+/wn870dAwF/ici8/h3ovwNaJjc0YxrUeQFVnpifsWLzza5iFVJ\nNSYVcQvyRKjq/kDHYkwgWEvBGGNMCmspGGOMSWEtBWOMMSksKRhjjElhScEYY0wKSwrGGGNSWFIw\nxhiTwpKCMcaYFP8PaEQwXmWAQbAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1ad803b14e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "logit_model = LogisticRegression(n_jobs=1,max_iter=50,\n",
    "                                 class_weight={0:1, 1:1.1})\n",
    "t_init = time.time()\n",
    "logit_model.fit(X_train, y_train)\n",
    "t_final = time.time()\n",
    "print(\"Total fitting time: {0}s\".format(t_final-t_init))\n",
    "pred_logit_test = logit_model.predict(X_test)\n",
    "pred_logit_test_proba = logit_model.predict_proba(X_test)[:,1]\n",
    "fpr, tpr, _ = roc_curve(y_test, pred_logit_test_proba, pos_label=1)\n",
    "df = pd.DataFrame(dict(fpr=fpr, tpr=tpr))\n",
    "plt.plot(fpr,tpr)\n",
    "plt.plot([0,1],[0,1],'r--')\n",
    "plt.ylabel('True positive rate')\n",
    "plt.xlabel('False positive rate')\n",
    "print(\"roc_auc score: {0}\".format(roc_auc_score(y_test,pred_logit_test_proba)))\n",
    "print(\"Total positive values on test set: {0}\".format(y_test.sum()))\n",
    "print(\"Total positive predictions on the test set via LogisticRegression: {0}\".format(pred_logit_test.sum()))\n",
    "print(\"There are {:.2f}% of positive values on the prediction test\".format(100*pred_logit_test.sum()/len(test)))\n",
    "\n",
    "pred_logit_valid = logit_model.predict(data_valid[features])\n",
    "print(\"This model predicts {0:.2f}% of positive results on validation set\".format(100*pred_logit_valid.sum()/len(data_valid)))\n",
    "pred_logit_proba_valid = logit_model.predict_proba(data_valid[features])[:,1]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RandomForest classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting train set with \n",
      " n_estimators = 50 \n",
      " min_samples_leaf = 120 \n",
      "Total time: 24.43090319633484s\n",
      "Train score: 0.377713933810264\n",
      "Test score: 0.03604579857698479\n",
      "119043.0\n",
      "AUC-ROC: 0.5\n",
      "\n",
      "Predicting on test data...\n",
      "The Random Forest model yields : \n",
      "Train score = 0.377713933810264\n",
      "Test score = 0.03604579857698479\n",
      "AUC_ROC = 0.5\n",
      "roc_auc score: 0.6289136042727077\n",
      "Predicting on validation data ...\n",
      "This model predicts 100.00% of positive results on the validation data\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XmczeUXwPHPMWMnO2Udsu8x9rJE\n0kYrKkqJJO17WiQpLZQiFFHSnlIp+Skp+75vY98ZjN2Y5fz+eK4xaZaLuffOvXPer9d9+d7v/d77\nPV/DPfN8n+c5j6gqxhhjDEC2QAdgjDEm87CkYIwxJoklBWOMMUksKRhjjEliScEYY0wSSwrGGGOS\nWFIwxhiTxJKCMcaYJJYUjDHGJAkPdADnqmjRohoRERHoMIwxJqgsXLgwWlWLpXdc0CWFiIgIFixY\nEOgwjDEmqIjIFm+Os9tHxhhjklhSMMYYk8SSgjHGmCSWFIwxxiSxpGCMMSaJz5KCiIwRkb0isiKV\n10VEhopIlIgsE5F6vorFGGOMd3zZUhgLtEvj9WuASp5HT+BDH8ZijDHGCz6bp6CqM0QkIo1DOgCf\nqlsPdI6IFBSRS1R1l69iMsaYYKKq7Ig5wcylWzm2fSeRLetRu3RBn54zkJPXSgHbkj3f7tn3n6Qg\nIj1xrQnKli3rl+CMMSYQVu48xE9Ld/HD4h3sPnySJluW8sZv73MkZ16WfD81pJOCpLBPUzpQVUcB\nowAiIyNTPMYYY4LN/qOxTFq6kz/W7CUhUVmw5SCn4hMBuERP8tHMcVw18yeOlS1PyU8+ombT8j6P\nKZBJYTtQJtnz0sDOAMVijDE+paqs33uUP9bs5Z/10WyKPsaOmBP/Ouaq6iWoWDwfN9QoQfWrm8Ha\ntfD00+Tt1w9y5/ZLnIFMCpOAPiLyJdAIOGT9CcaYUJGYqMzcEM0/UdFs3X+cORv3c/B4XNLrxfLn\n5PKKRel+eXmaXFqEXNnDYP9+KFwYROC116BMGYiM9GvcPksKIvIF0BIoKiLbgZeB7ACqOgKYDFwL\nRAHHgXt8FYsxxviaqrJkWwxTVu5hc/QxZkZFcyQ2Pun15pWL0aRCEWqWuojLKxZFRJK/GcaPh0ce\ngTfegB494KabAnAVvh19dHs6ryvwoK/Ob4wxvqaqTF21h4//2cS8TQf+9VrNUhdxfe2SXFm1OJWK\n5/t3Ekhu2zbo1QsmT4bGjaFZMz9EnrqgK51tjDGBpKos3hbDpCU7+WbBNo6dSgCgdKHcdIwsQ/s6\nJSlXJE/qSSC5L76A+++HhAR4913o0wfCwnx8BWmzpGCMMek4FZ/Igi0HWLHjEL8s28XS7YcAaFS+\nMO1qXkyHuqUonDfHuX9woULQqBGMGgXlfT+yyBuWFIwxJgXHT8Xz4fQNzNm4n/mbD/7rtU6RZehz\nZUXKFM5zbh8aHw9DhsCpU9C3L7RrB1df7TqWMwlLCsYY46Gq/LRsF2P+2cSSbTFJ+6+oVJTmlYrR\nrGJRKhbPR47w86gQtHQpdO8OCxdCx46uc1kkUyUEsKRgjMniTsYlMPqfTXyzYBub9x9P2l+7dAE6\nNSjD7Q3Kki3bBXxxx8bCgAFuVFHhwvDNN3DLLZkuGZxmScEYk+XsPXKSSUt2MmN9NDPW7UvaH1mu\nEK2qFqdb0wjy5sygr8f162HQILjjDhg8GIoUyZjP9RFLCsaYLGPfkVie+nYp09eeSQTX1bqEVlWL\nc0u9Ut6NGPLG0aPw449w551QsyasWQMVKmTMZ/uYJQVjTMhSVeZvPshvK3YzMyqatXuOANCySjE6\nNyhL2+olLuzWUEqmToWePWHLFqhXD6pVC5qEAJYUjDEhKC4hkZF/bWDsrM1EHz2VtP/yikXp3epS\nml5aNONPevAgPPkkjBkDlSvDX3+5hBBkLCkYY0LCyp2HmLJyD3+u2cvyHW4eQZ4cYbxwXTXa1ylJ\n8Yty+e7kCQluJvK6dfDcc/DSS5DLh+fzIUsKxpigparMWB/N0GnrWbjlzFyCNtVK0KFuSa6peTHh\nYT5cYDI62o0oCguDgQOhbFl3yyiIWVIwxgSlBZsP8OhXS9h+8AQicGv90nRrGkGNkhdlXIdxalTh\ns8/g0UfdUNOePeHGG317Tj+xpGCMCSqHT8YxZOo6Ppm5GYD7m1fg4daVMm4IaXq2bHH1iqZMgaZN\noXlz/5zXTywpGGOCwox1+xg+PYo5G1010jbVStD3umqUL5rXf0GMHw8PPOBaCu+/D717QzYf3p4K\nAEsKxphMSVU5eDyO31bs5tcVu/h7fTQAV1Ytzv3NK9CwfGHf3yY6W7FirkN55EgoV86/5/YTSwrG\nmExDVZm36QAT5m1l8vJdxCWcWZL9nmYRPHxlJQqdTzXS8xUXB++84/588UVXvK5t20xboiIjWFIw\nxgScqjLkf+sZOm190r6i+XJwV5MISlyUk2trXUL+XNn9G9Tixa6A3eLF0Llzpi1gl9EsKRhjAmZH\nzAl+Xb6L96at58hJt3TlnY3Kck+zCCoWzx+YoE6ehP794c03oWhR+O47uPnmwMQSAJYUjDF+9+vy\nXbz1+1o27jsGQPH8Obmtfhmevabq+ZWlzkhRUfD223DXXe7WUaFCgY3HzywpGGP8QlWZvHw342Zt\nZt5mN4KoddXi9G5VkfrlAvzFe/QoTJwIXbu6AnZr12aaldD8zZKCMcandsacYMLcrYycsSGp4/iK\nSkV5+7Y6lPBl6QlvTZniJp9t2waRka5eURZNCGBJwRjjI9sPHqfTyDnsiDkBQN0yBWlVpTi3NypD\n8fyZIBns3w+PPw6ffgpVq8LffwdlAbuMZknBGJOhtuw/xkd/b2T8nK0ANIwozCNtKtGsog8qk56v\n0wXsoqLcWskvvBC0BewymiUFY0yGOH4qngG/rGbCXE8yKF+YV9rXoNolFwU4smT27XMrn4WFudXQ\nypWDunUDHVWmYknBGHNBZm2I5pVJq5IWsMmdPYwRXevTonKxAEeWjCqMHetuF73xhqtd1KFDoKPK\nlCwpGGPOWUKiMmfjfn5ZviupZVD14vx0aVyOOxuV9X/5ibRs3uw6kqdOhSuugFatAh1RpmZJwRjj\ntcREZcj/1vH+H1FJ+1pWKcaAG2tSulCeAEaWis8+cwXsRGD4cNdCCLECdhnNkoIxJl0nTiXw8d8b\neWfquqR9D11ZkZvrlfZvldJzVaKEK209YoRbAMeky5KCMSZVk5buZMLcLUnlqksVzE3zykV58frq\n5MmRCb8+4uJceYqEBLckZtu27mG8lgl/qsaYQFJVxs3azOiZm9h2wM0xaFOtOK2rleD2hpn4t+1F\ni+Dee2HpUrjjjjMF7Mw58WlSEJF2wHtAGPCxqr5x1utlgXFAQc8xz6rqZF/GZIxJ2bHYeAZPXcfv\nq3YnJYOOkaV5pX1NcucIC3B0aThxAl55xdUrKlbMlasIkaUxA8FnSUFEwoBhwFXAdmC+iExS1VXJ\nDnsB+FpVPxSR6sBkIMJXMRlj/mvB5gN8NX8bvyzfxfFTCeTJEcYz7apyV5Ny/lvi8kJs3AiDB0O3\nbvDWW1mugF1G8+VPvCEQpaobAUTkS6ADkDwpKHB6ZksBYKcP4zHGJBMbn8DgqesY+ddGALKHCY9f\nVZmHW1cKcGReOHwYvv/eJYIaNWD9+pBdCc3ffJkUSgHbkj3fDjQ665h+wO8i8hCQF2jjw3iMyfLi\nExKZs/EAg35bw/IdhwCoUDQvH3apT5WLA7R+wbmaPBl69YIdO6BRI1evyBJChvFlUkiph0fPen47\nMFZV3xGRJsBnIlJTVRP/9UEiPYGeAGVtWJkx5+Wvdfu4e8y8pOfliuThuWuq0q7mJQGM6hxER8Nj\nj8H48VC9OsycaQXsfMCXSWE7UCbZ89L89/ZQd6AdgKrOFpFcQFFgb/KDVHUUMAogMjLy7MRijEmD\nqjJm5mZe/XkVhfPmoGNkGe5uWo5LCuQOdGjeO13AbuNGN9T0+echZ85ARxWSfJkU5gOVRKQ8sAPo\nDNxx1jFbgdbAWBGpBuQC9vkwJmOylEVbD3LfuAUcOHaKKiXy80XPxhT258L3F2rPHjeiKCzMjS4q\nVw5q1w50VCHNZ/O9VTUe6ANMAVbjRhmtFJH+ItLec9gTQA8RWQp8AXRTVWsJGHOBVJXBU9dx24jZ\nHD8Vz1NXV+G3R68InoSgCqNHQ5UqMGqU23fDDZYQ/MCn4808cw4mn7XvpWTbq4BmvozBmKwkLiGR\n4X9uYNzszRw4dor65QrxYZd6mWNRG29t3Ag9esAff0CLFtDGxp/4UxAMQjbGpOfQ8ThemrSCH5e4\nbrsKRfNyT9MI+lxZMXNVLE3PuHHQu7e7XTRihEsOVsDOrywpGBPEth04Tr9JK5m2xo3NuPiiXPRo\nXoHulwfpGsMlS8KVV8KHH0Lp0oGOJkuypGBMENp16ASjZmzks9lbiE9UapUqwCOtK9GmeolAh3Zu\nTp1yi94kJkK/fnDVVe5hAsaSgjFB5PeVuxnx1wYWbY0BoMRFORnSsS5NM9P6x96aP98VsFuxArp2\ntQJ2mYQlBWOCwOGTcTzx9VKmrtoDwLW1LuaeZuVpEFE4wJGdh+PH3VyDIUPgkktg0iQ3sshkCpYU\njMnEDp+Mo+/EFfy01HUgt6txMYNuqU2BPNkDHNkF2LQJ3n/fdSIPGgQFCgQ6IpOMJQVjMqGTcQkM\nnbaez2Zv4UhsPBFF8vDk1VW4vnbJQId2fg4dcgXs7rnHFbCLioIyZdJ/n/E7SwrGZDKzN+znqW+X\nsv3gCSKK5GF0twY0LB+Et4lO++UXtzbyrl3QpAlUrWoJIROzpGBMJrH70EnGztrMiL82kDM8G+92\nqkuHuiWDa55Bcvv2waOPwoQJULOmaylUrRroqEw6LCkYE2C7Dp2g78QV/OGZa9CofGE+7FI/eEpS\npCQhAS6/3PUfvPIKPPss5Aji68lCLCkYEyBLt8Xw8T+bkjqR8+YI463b6nBtrSApZZ2S3buheHE3\nI/mddyAiwrUSTNCwpGBMAEyYu5XnJy4HoFapArxwXTUaVSgS4KguQGIifPQRPPWUG1H0wANw/fWB\njsqch3STgojkBh4FyqlqLxGpCFRS1V99Hp0xIWbL/mN8MnMzY2dtpnDeHIy7pyG1Sgf5kMyoKDe8\ndPp0V6Li6qsDHZG5AN60FMYAy4HLPc93At8AlhSM8VL00Vh6j1/EvM0HAKhfrhCjutanSL4gXyjm\nk09cAbscOVxLoXt3m5Uc5LxJCpVU9XYRuQ1AVY9L0A6HMMa/jsXG89hXS5i2Zi8JiUqrKsV4om0V\napYK8tbBaWXLupbBsGFQqlSgozEZwJukcMqzTKYCeFZSO+XTqIwJAcu2x9Dz04XsPnySJhWK0Pe6\nasGfDGJj4fXXXR9C//7QurV7mJDhTVJ4FfgNKC0i44AWwH0+jcqYIHXoeBz/REXzzcJtTF/rVpZ9\n6frq3BuspayTmzvX3R5auRLuvtsK2IWodJOCqv4qIguApoAAT6nqXp9HZkyQUFV+XbGb9/+IYvWu\nw0n7SxfKzXudL6N+uUIBjC4DHDsGL74I777rbhH9/DNcd12gozI+4s3oo99VtS3wYwr7jMnSdh06\nQe/PF7HYU8q6col8PNy6EnVKF6RM4TwBji6DbNkCw4dDr15u7YOLLgp0RMaHUk0KIpIDyAWUEJH8\nuFYCwEVAWT/EZkymFX00lke/XMI/UdEA3HxZKV5uX4MCuYO4emlyMTHw7bdw331QvbobdmoroWUJ\nabUUHgQeB4oDKzmTFA4DI3wclzGZ1uTlu+j9+SLAtQwG3lSLyGBc1yA1P/7oJp/t3etKVVStagkh\nC0k1KajqEGCIiDyqqu/6MSZjMqWTcQncN24B/0RFU6pgbgbcVJNWVYoHOqyMs3cvPPwwfPUV1K7t\nFr+xAnZZjjcdze+KSFWgOu520un9E3wZmDGZxbHYeEbN2MiH0zdwKiGROmUK8mWPxuTOERbo0DJO\nQgI0awZbt8KAAfD005A9RG6FmXPiTUfzC0BboCowBbga+AewpGBC3qKtB+n56QKij56iUJ7sPHJF\nJXq3vDR4y1mfbedOuPhiV8DuvfdcAbvq1QMdlQmgbF4c0wloBexS1a5AHayQnskCpqzczc3DZxF9\n9BT3N6/Aohev4sFWFUMjISQmwocfuttDIzxdhNdeawnBePXlfkJVE0Qk3jMKaTdQwcdxGRMQR07G\nMXDyGn5ZtpPDJ+MpVTA3I7vWD/6ZyMmtW+cK2M2YAW3awDXXBDoik4l4kxQWi0hBXGG8BbjRR4t8\nGpUxATBv0wF6f76Q6KOniCxXiJqlCvBMu6qh1XcwejT06QO5csGYMdCtm81KNv+SZlLwFL7rp6ox\nwDARmQJcpKqWFExIWbfnCB1Hzgbgxeur0z0UylKkJCLCtQyGDYNLgngxH+MzaSYFVVUR+Rmo73ke\n5ZeojPGTxESl308r+XrBNnJlz8b47o1Ca85BbCy8+qrbHjDACtiZdHlz+2ieiNSz1oEJNcu2x/DA\n+EXsiDkBwDe9moRWQpg1yxWwW7MG7r3XCtgZr3iTFC4HeojIBuAYbmazqmo9n0ZmjA/9b9UeHv5y\nMSfjEnj1xprc2bAs2bKFyBfm0aPQty+8/z6UKQO//WaroRmveZMUbjzfDxeRdsB7QBjwsaq+kcIx\nHYF+uPUalqrqHed7PmPS8+W8rbz2y2qOxMaTL2c4vz3anMol8gc6rIy1dSuMHAkPPggDB0L+ELs+\n41PezGjecD4fLCJhwDDgKmA7MF9EJqnqqmTHVAKeA5qp6kERCaGaASYz2br/OH1/WM7f610Bu2YV\ni/Be58soGuzLYZ528CB88w307OnmGmzcCCVLBjoqE4R8OQmtIRClqhsBRORLoAOwKtkxPYBhqnoQ\nwNZpMBltZ8wJnvt+OX+tcwve3FCnJG/dWptc2UNomOnEiW6d5H37oEULqFLFEoI5b75MCqWAbcme\nbwcanXVMZQARmYm7xdRPVX87+4NEpCfQE6BsWavabdKnqnyzcDsv/LCCU/GJtKhcjKfbVaFGyRCa\nhLZ7Nzz0kCtxXbcu/PKLSwjGXACvkoKIlAYqqeqfIpITCFfVY+m9LYV9msL5KwEtgdLA3yJS0zMv\n4sybVEcBowAiIyPP/gxj/uVkXALdx81nZtR+ShbIxXu3X0aDUBpVBK6A3RVXwLZtrt/gySetgJ3J\nEN4UxLsX6AMUAC4FygHDgTbpvHU7UCbZ89LAzhSOmaOqccAmEVmLSxLzvYremLMkJip9JixiZtR+\nbm9YhpdvqBFat4q2b3e3hsLCYOhQKF/eylubDOVNQbyHgca48hao6jrcwjvpmQ9UEpHynlXcOgOT\nzjrmB1yxPUSkKO520kbvQjfm36L2HuXKd6bzv9V76dq4HK/fHEJ9B4mJbohp1aqukB24mcmWEEwG\n8+b20UlVPXW6MqRnVFG6A7pVNV5E+uDKbYcBY1R1pYj0Bxao6iTPa21FZBWQADylqvvP81pMFnUy\nLoGBk1fz6ewthGUTelxRnuevrRbosDLOmjVuWcyZM918g+uvD3REJoR5kxRmisjTQC4RaYVbpvNn\nbz5cVScDk8/a91KybcUt+fm41xEbk8zf6/fx2FdLiT4aS72yBRnSqS7liuQNdFgZ5+OPXQG7PHlg\n3Djo2tVmJRuf8iYpPI0b+bMGeAT32/1IXwZljDfGzdrMy5NWkiM8G/1uqM7dTSNCY62D5C69FG64\nAT74AEqUCHQ0JgvwJilci5uN/KGvgzHGGyfjEnh+4nK+X7SDSsXzMfHBZuTLGSLrPp08Cf37u+2B\nA6FVK/cwxk+86WjuCESJyCcicrWnT8GYgNi47ygdR87m+0U7uLV+aX7sE0IJYeZMN9/g9dfdRDS1\n0dfG/7wpc9HVMzfhOuBeYJSI/KqqvXwenTEeiYnKu9PWM3TaegDeurU2t0WWSeddQeLIEXj+ebfG\nQblyMGUKtG0b6KhMFuXVr1iqGisiPwIncCOJOgKWFIxfbDtwnN6fL2L5jkPUL1eIN2+tzaXF8gU6\nrIyzfbvrUH7oIXjtNcgXQtdmgo43k9fa4OYYtAFmAp8CVsnU+MWsDdHc8dFcAJ64qjJ9rqwYGp3J\n+/fD11/DAw9AtWqugJ2thGYyAW9aCr2AL4GHVPWEj+MxBnC1i97/I4rBU9cRnk0Y0aU+baqHwOgb\nVfjuO1fW+sABuPJKV6/IEoLJJLzpU7jVH4EYc9reIyfp+vE81u45Qv6c4Ux7sgXF8+cKdFgXbtcu\nlwwmToT69eH3362Ancl0Uk0KIvKXqrYQkYP8u5Dd6ZXXQqzCmMkM/lq3j7vHzAPg1vqlGXhTLXKE\nezNILpM7XcBuxw5480147DEID5FRUyakpPWv8vTg6KL+CMRkbYdPxvHmb2sYP2crubOH8cYttehQ\nt1Sgw7pw27ZBqVKugN2wYa6AXeXKgY7KmFSl+iuYqiZ6NkerakLyBzDaP+GZrOCjGRup3e93xs/Z\nSsPyhfnjyRbBnxASElwV0+QF7K6+2hKCyfS8ab/WTv7EM3mtgW/CMVlJXEIiD3+xmF9X7KZUwdz0\na1+Dq0KhM3n1aujeHWbPdpVMb7gh0BEZ47W0+hSeAZ4F8ovIgdO7cf0L1lIwF2TfkViuHfo3+47E\ncmPdkgy6tTY5w0NgsvyoUW6+Qf788NlncOedVsDOBJW0WgpvAu8Ar+OSAwCe20fGnBdVZfzcrbz4\nwwoA7m9egedCqcx1pUpw003u1lFxb5YdMSZzSSspVFTV9SLyGVDj9M7TE4dUdZmPYzMhJi4hkae+\nWcoPS3ZSME923rmtDq2rBfntohMnoF8/1xp44w0rYGeCXlpJ4VmgOzAshdcUaO6TiExImrR0Jy/9\nuIKY43HUK1uQL3o2Dv7bRTNmuMVv1q+HXr3cxDS7VWSCXKpJQVW7e/68wn/hmFBzNDaeAT+v4sv5\n2yiSN0doFLI7fBiefdaNKqpQAaZNczOTjQkB3tQ+uhmYqqpHRORZoB7wmqou9Xl0Jqht2HeU20fN\nYe+RWK6peTHvdq4b/K0DgJ07YexYePxxt/ZB3hBa6c1ked4MSe2nqt+LSFPgBmAwbuW1xj6NzASt\nQyfiGD49ipF/bSSbwBs316Jzw7KBDuvCREe7Ana9e7u5B5s22UpoJiR5kxROjza6Hhiuqt+JyAs+\njMkEsfmbD/DQhMXsPnyS4vlzMuquSOqWKRjosM6fqksGDz0EMTHQpo2bgGYJwYQob5LCLhEZBlwD\n1BeRHHi3YpvJQlSVYX9G8fbv68gRno33OtflhtolyZYtiDted+50pa0nTYLISNd3YDOSTYjzJil0\nxK3T/L6qHhSRkiSbt2DMgs0HePzrpWw9cJySBXIx6aHLKZovZ6DDujAJCdC8uStg9/bb8MgjVsDO\nZAnelM4+KiKrgJYi0hL4W1V/9XlkJih8OH0Dg35bA8AdjcrySvsaZA8L4obkli1QurQrYDd8uBtd\nVLFioKMyxm/S/d8rIn2Ar4GynsfXItLb14GZzG33oZNcN/RvBv22hogieZj3fGsG3lQreBNCQgIM\nHuxWQTtdwK5tW0sIJsvxpj3cE2ioqkcBRGQgMAsY7svATOY1b9MB+kxYxN4jsXRtXI4Xrq8W3ENN\nV6xwBezmzYPrr4cbbwx0RMYEjDdJQYC4ZM/jPPtMFrRo60Hu+GgOOcOzMb57Iy6vFOTLbYwYAQ8/\nDAUKwIQJ0LmzzUo2WZo3SeEzYI6IfIdLBjcC43walcl0Fm89yNBp6/lz7T7CsgkTH2xG5RL5Ax3W\n+TtdkqJaNbjtNnj3XShWLNBRGRNw3nQ0vykifwKny130UtX5vg3LZCZj/tlE/59XAVCxeD6Gdr4s\neBPC8ePw0kuuI3nQIGjRwj2MMYB3LQWAWM8j0fOnySJ+WbaL/j+v4uKLcvHJPQ2odslFgQ7p/E2f\n7grYbdjgZiZbATtj/sOb0Ud9gS+AS4DSwAQRec7XgZnA+3bhdh6csIhSBXMz7YkWwZsQDh2C++8/\nU9L6jz/cesmWEIz5D2/GD3YBGqjqC6raF2gI3OXNh4tIOxFZKyJRnmJ6qR13q4ioiER6F7bxpVPx\nibzww3Ke/GYphfPm4JN7GpA3ZxBP3Nq1C8aPhyefhGXLbL0DY9Lgzf/0LWcdFw5sTO9NnrWchwFX\nAduB+SIySVVXnXVcfuBhYK63QRvf2RFzgjs+msOW/cepWeoivujRmPy5sgc6rHO3bx98+aWrWVS1\nKmzebB3JxnjBm5bCcWCliHwsIh8By4EYERksIoPTeF9DIEpVN6rqKeBLoEMKx72KW/rz5DnGbjLY\nrA3RXPn2dLYeOM4NdUryU5/Lgy8hqLqhpdWqwRNPwLp1br8lBGO84k1L4RfP47Q5Xn52KWBbsufb\ngUbJDxCRy4AyqvqziDzp5eeaDLZx31EGTl7D/1bvIX/OcL66v0lwVjbdts0VsPvlF2jUCEaPtgJ2\nxpwjb4akjj7Pz06pF0+TXhTJBgwBuqX7QSI9cTOrKVs2yOvyZzK/LNvFs98v48jJeCLLFWLQrbW5\ntFi+QId17uLjoWVL2L0bhgxxt43CgniWtTEB4svew+1A8nUXSwM7kz3PD9QEposbBXIxMElE2qvq\nguQfpKqjgFEAkZGRirlgqsrbv69l2J8bCM8mTOrTjNqlg7B1sHkzlCnjKpiOHOkK2FWoEOiojAla\nvqxeNh+oJCLlPWswdAYmnX5RVQ+palFVjVDVCNxtqf8kBJPxEhKVp79dxrA/N1C/XCHmPt86+BJC\nfLwraV2tmqtmCm4BHEsIxlwQr1sKIpJTVb2euKaq8Z4Kq1OAMGCMqq4Ukf7AAlWdlPYnGF+IjU+g\n9/hFTFuzl5suK8Xbt9UhLNgWwlm2zBWwW7AAOnSAW24JdETGhIx0k4KINARGAwWAsiJSB7hPVR9K\n772qOhmYfNa+l1I5tqU3AZvzt/vQSR74fCGLt8Zwf4sKPHdNtUCHdO6GD3cL3hQqBF995eoW2SQ0\nYzKMN7ePhuLWZ94PoKpLAZv9E2TiExLp7UkIz7SrGnwJQT1dSTVrukqmq1ZBx46WEIzJYN7cPsqm\nqlvk3//5EnwUj/GBU/GJPPELv03WAAAWo0lEQVTNUhZtjaHfDdXp1qx8oEPy3rFj8MILriP5rbfc\nEpnNmwc6KmNCljcthW2eW0gqImEi8iiwzsdxmQxyKj6RB8Yv5KelO7mjUVnubhoR6JC8N20a1Krl\nylrHxp5pLRhjfMabpPAA8DhuKc49QGPPPhMEHvtqCdPW7KVb0wgG3lQLCYbbLTExrpppmzauhTBj\nBgwdareKjPEDbyav7cUNJzVBZNehE/T8dCHLdxzilnql6de+RqBD8t6ePa5u0TPPwMsvQ+7cgY7I\nmCzDm9FHH5FsJvJpqtrTJxGZC7Z+zxG6jp7H7sMnuaVeafp3CIKEcDoRPPIIVKniJqUVDfKlPo0J\nQt50NP8v2XYu4Cb+XdPIZCJzN+6n0yhXnuqTexrQqkrxAEeUDlX4/HOXDI4ehWuvhUqVLCEYEyDe\n3D76KvlzEfkMmOqziMx5OXIyjrEzNzNsehT5coYz9p4GREYUDnRYadu6FXr1gl9/hSZNXAG7SpUC\nHZUxWdr51D4qD5TL6EDM+YlPSOSDP6N493/rAcgZno2fH7qciKJ5AxxZOk4XsNu713Ui9+5tBeyM\nyQS86VM4yJk+hWzAASDVVdSM/xw4doquo+eycudhyhbOwyOtK3F9nUvIGZ6Jv1w3boRy5dyooo8+\ngksvhYiIQEdljPFIc0iquPGLdYBinkchVa2gql/7IziTutj4BO4a4xJCl8Zl+d/jLbilfunMmxDi\n42HQIKhe3a2PDNC6tSUEYzKZNFsKqqoiMlFV6/srIJO+LfuPcdeYeWzZf5ynrq7Cg60qBjqktC1Z\n4grYLVoEN93k6hUZYzIlbyavzRORej6PxHhl3Z4jtHhrOlv2H+exNpUzf0L44ANo0AB27IBvv4Xv\nv4dLLgl0VMaYVKTaUhCRcFWNBy4HeojIBuAYbkU1VVVLFH625/BJeny6gPBswpu31ubmeqUDHVLq\nVN0M5Nq14c47YfBgKJzJR0MZY9K8fTQPqAfc6KdYTBoWbz1Iz88Wsu9ILCO61KNdzUz62/bRo9C3\nL2TP7hbBsQJ2xgSVtJKCAKjqBj/FYlKxZvdhunw8FxHhix6NaXJpkUCHlLLff4eePd38g4ceOtNa\nMMYEjbSSQjEReTy1F1V1sA/iMWdZveswXUfPQ0T47oGmVLk4f6BD+q+DB+Hxx2HsWFeiYsYMuPzy\nQEdljDkPaSWFMCAfnhaD8b8VOw5x47CZZMvmWgiZMiGAm4D27bfw3HPw0kuQK1egIzLGnKe0ksIu\nVe3vt0jMvyzdFkPHkbPJEZ6Nib2bZb6EsHs3fPEFPPbYmQJ2RTLpbS1jjNfSGpJqLYQAUFVG/rWB\nW0fMIi4hkXc71c1cCUEVxo1zk9Ceew7Wu/IalhCMCQ1pJYXWfovCJOk3aSWv/7oGQZjyaHPa1rg4\n0CGdsXkztGsH3bq5pLBkiRWwMybEpHr7SFUP+DMQAyP/2sC42VtoX6ckb9xSizw5zqdeoY/Ex0Or\nVhAd7cpU9OoF2byZ+2iMCSaZ6Fsn61JVPp29hdd/XUPF4vl4+7Y65AjPJF+4UVFQvrwrYDdmDFSo\n4AraGWNCUib55snahvxvPS9PWknRfDmY0KNR5kgIcXEwcCDUqHGmgF2rVpYQjAlx1lIIIFVlyNR1\nDP0jiqoX5+eHB5uRK3smqHK6aJErYLdkiSte16lToCMyxvhJJviVNOsa8Mtqhv4RRe3SBfjugaaZ\nIyEMHQoNG7ohp99/D19/DSVKBDoqY4yfWFIIAFVl4OTVjP5nE9fVuoQfH2xG3pwBbrSpZx2lyy6D\nu+6CVatcmWtjTJZit48C4JnvlvH1gu20rFKMIZ3qIoGsD3TkiJtvkDMnvPMOXHGFexhjsiRrKfhZ\nv0krkxLC6LsbBLZT+bffoGZNGD7ctRROtxaMMVmWJQU/UVVe/nEFY2dtplH5wnx0VyRh2QLUQti/\nH+6+G665BvLmhZkz3XoHVtHUmCzPbh/5QczxU9w7dj6LtsZwQ52SDO5Yh+xhAczH+/fDxInw4otu\n7YOcOQMXizEmU/HpN5OItBORtSISJSLPpvD64yKySkSWicg0EQm5QfC7D52k48jZLNoaQ8sqxQKX\nEHbtcoveqELlyrBlC/TvbwnBGPMvPmspiEgYMAy4CtgOzBeRSaq6Ktlhi4FIVT0uIg8AbwIhMyh+\n4ZYD3DZiNokK799+GTfUKen/IFThk0/cegexsdChg6tXVKiQ/2MxxmR6vvyVtSEQpaobVfUU8CXQ\nIfkBqvqnqh73PJ0DZOJFh8/NrkMn6PnpQsKzZeOrno0DkxA2bYK2bd1EtDp1YOlSK2BnjEmTL/sU\nSgHbkj3fDjRK4/juwK8pvSAiPYGeAGXLls2o+HzmaGw8d42ex6ETcXzTqwmXlQ3Ab+Xx8XDlla7/\n4MMP3TKZVsDOGJMOXyaFlIaypDjmUUS6AJFAi5ReV9VRwCiAyMjITD1uUlW5d+x81u89yoAba/o/\nIaxf74rWhYe720aXXgplyvg3BmNM0PLlr47bgeTfRqWBnWcfJCJtgL5Ae1WN9WE8fjFh3lbmbTrA\n/S0q0KWxH/vN4+JgwAA37+CDD9y+li0tIRhjzokvWwrzgUoiUh7YAXQG7kh+gIhcBowE2qnqXh/G\n4hezoqLpO3EF5Yvm5am2Vfx34gULXL/BsmXQuTPcfrv/zm2MCSk+aymoajzQB5gCrAa+VtWVItJf\nRNp7DnsLyAd8IyJLRGSSr+Lxtai9R7nv0wWIwLh7GhLur2Gn770HjRq5xW9+/NGtm1y8uH/ObYwJ\nOT6dvKaqk4HJZ+17Kdl2G1+e31+OxcbTZ8IiEhKVH3o3o2yRPL4/qaqbgRwZ6VoJb74JBQv6/rzG\nmJBmM5ov0Mm4BLqOnsua3UcYdkc96pTx8Rfz4cPwzDOQKxcMGQLNmrmHMcZkABujeIEGT13Hoq0x\nPHV1Fa6rfYlvTzZ5slsJbdQoN7rICtgZYzKYJYULsHDLQUbN2EjTS4vwYKuKvjtRdDR06QLXXQcF\nCsCsWfDWW1bAzhiT4SwpnKc5G/dzy4ezyJU9GwNvquXbkx08CD/9BC+/7JbKbJTWHEBjjDl/1qdw\nHrYfPE6PcQsomCc7w++sR0TRvBl/kh074PPP4amnXGmKLVusI9kY43PWUjhHcQmJ9JmwmFMJiXxz\nfxOaXlo0Y0+gCh99BNWrQ79+sGGD228JwRjjB5YUztHwPzewZFsMz11TlUol8mfsh2/YAK1buzpF\n9eq5yWgVfdhXYYwxZ7HbR+dgxrp9vP/HeppVLEK3ZuUz9sPj411COHAARo6E++6zAnbGGL+zpOCl\ndXuO0POzBVxcIBdDOtXNuA9eu9YVrQsPh3Hj3HbpkKkgbowJMvarqBdUlRd/WEGYCJ/e25Di+XNd\n+IeeOgWvvAK1asGwYW5fixaWEIwxAWUtBS98/Pcm5m46wLPXVKVCsXwX/oHz5rnSFCtWwB13wJ13\nXvhnGmNMBrCWQjpmbYjmtcmrqVOmIPddngH9CO++C02anJl78PnnUDSDRzAZY8x5sqSQhm0HjtPl\n47kUz5+TT++9wMqnp0tSNGwIPXrAypVw/fUZE6gxxmQQu32Uhue+X06iwrud6lIgd/bz+5BDh+Dp\npyF3btdKaNrUPYwxJhOylkIqZqzbxz9R0TzSuhJNK57n7Z2ffnKT0D7+GHLmtAJ2xphMz5JCCmLj\nE3jq26WUKpibB1peeu4fsG+f60Bu3x6KFIE5c2DQICtgZ4zJ9CwppODVn1ex53AsL15fjVzZw879\nAw4dcmWuX3nFLZXZoEHGB2mMMT5gfQpnWbT1IOPnbOXmeqVoV/Mc1kfYtg3Gj4dnn3WlKbZscWWu\njTEmiFhLIZmovUfoNHI2OcKz8fL1Nbx7U2IijBjhFr8ZMOBMATtLCMaYIGRJwSMxUXnu++XEJSgf\n3RVJgTxejDZavx6uvBIeeMANNV2+3ArYGWOCmt0+8hgzcxPzNx/kheuq0aJysfTfEB8PV10FMTEw\nejTcc491JBtjgp4lBSBq71Fem7yaiCJ5uLtpRNoHr17tFr0JD4fPPnMF7EqW9Eucxhjja1n+9lF8\nQiKPfrWYfDnDGX9fI7KnNms5NtYth1m7Nnzwgdt3xRWWEIwxISXLtxQGT13Hih2HeePmWpQulCfl\ng+bMcQXsVq2Crl3dwxhjQlCWbinMiopm+PQNNK9cjE4NyqR80DvvuLIUR464uQeffuompBljTAjK\nskkhIVF547c1FMyTnQ/vrIec3UmcmOj+bNIEevVyZa6vucb/gRpjjB9l2dtHz3y3jGXbDzHgxprk\nzZnsryEmBp54AvLkgffftwJ2xpgsJUu2FH5bsYtvF26ncYXC3Nmo7JkXfvjBFbAbNw7y57cCdsaY\nLCfLJYW9R07y/MQVXJQrnI/vbuBuG+3dCx07wk03QYkSbmW0gQNt3oExJsvJcklhwM+rOXDsFB/c\nUY98p28bHT4MU6fCa6+5hFCvXmCDNMaYAPFpUhCRdiKyVkSiROTZFF7PKSJfeV6fKyIRvoxn+8Hj\n/LRsJ7fUK03zXCdcElB1pSm2boXnn4fs57mYjjHGhACfJQURCQOGAdcA1YHbRaT6WYd1Bw6qakVg\nCDDIV/EADJm6njCUvlv+dAXsBg48U8Auf35fntoYY4KCL1sKDYEoVd2oqqeAL4EOZx3TARjn2f4W\naC3/GRuaMWKOn2LZtLn8/uPLFH76MTfUdOVKK2BnjDHJ+HJIailgW7Ln24FGqR2jqvEicggoAkRn\ndDBTl+7gky9f5GJi4ZNP4O67rSPZGGPO4suWQkrfuGeP8fTmGESkp4gsEJEF+/btO69gihTKw+cP\nvUa2VaugWzdLCMYYkwJfJoXtQPLaEaWBnakdIyLhQAHgwNkfpKqjVDVSVSOLFfOirHUKrqxagmcG\n3Ee2UlbAzhhjUuPLpDAfqCQi5UUkB9AZmHTWMZOAuz3btwJ/qNqMMWOMCRSf9Sl4+gj6AFOAMGCM\nqq4Ukf7AAlWdBIwGPhORKFwLobOv4jHGGJM+n9Y+UtXJwOSz9r2UbPskcJsvYzDGGOO9LDej2Rhj\nTOosKRhjjEliScEYY0wSSwrGGGOSWFIwxhiTRIJtWoCI7AO2nOfbi+KDEhqZnF1z1mDXnDVcyDWX\nU9V0Z/8GXVK4ECKyQFUjAx2HP9k1Zw12zVmDP67Zbh8ZY4xJYknBGGNMkqyWFEYFOoAAsGvOGuya\nswafX3OW6lMwxhiTtqzWUjDGGJOGkEwKItJORNaKSJSIPJvC6zlF5CvP63NFJML/UWYsL675cRFZ\nJSLLRGSaiJQLRJwZKb1rTnbcrSKiIhL0I1W8uWYR6ej5Wa8UkQn+jjGjefFvu6yI/Ckiiz3/vq8N\nRJwZRUTGiMheEVmRyusiIkM9fx/LRKRehgagqiH1wJXp3gBUAHIAS4HqZx3TGxjh2e4MfBXouP1w\nza2APJ7tB7LCNXuOyw/MAOYAkYGO2w8/50rAYqCQ53nxQMfth2seBTzg2a4ObA503Bd4zc2BesCK\nVF6/FvgVt3JlY2BuRp4/FFsKDYEoVd2oqqeAL4EOZx3TARjn2f4WaC0S1OtzpnvNqvqnqh73PJ2D\nWwkvmHnzcwZ4FXgTOOnP4HzEm2vuAQxT1YMAqrrXzzFmNG+uWYGLPNsF+O8Kj0FFVWeQwgqUyXQA\nPlVnDlBQRC7JqPOHYlIoBWxL9ny7Z1+Kx6hqPHAIKOKX6HzDm2tOrjvuN41glu41i8hlQBlV/dmf\ngfmQNz/nykBlEZkpInNEpJ3fovMNb665H9BFRLbj1m95yD+hBcy5/n8/Jz5dZCdAUvqN/+whVt4c\nE0y8vh4R6QJEAi18GpHvpXnNIpINGAJ081dAfuDNzzkcdwupJa41+LeI1FTVGB/H5iveXPPtwFhV\nfUdEmuBWc6ypqom+Dy8gfPr9FYothe1AmWTPS/Pf5mTSMSISjmtyptVcy+y8uWZEpA3QF2ivqrF+\nis1X0rvm/EBNYLqIbMbde50U5J3N3v7b/lFV41R1E7AWlySClTfX3B34GkBVZwO5cDWCQpVX/9/P\nVygmhflAJREpLyI5cB3Jk846ZhJwt2f7VuAP9fTgBKl0r9lzK2UkLiEE+31mSOeaVfWQqhZV1QhV\njcD1o7RX1QWBCTdDePNv+wfcoAJEpCjudtJGv0aZsby55q1AawARqYZLCvv8GqV/TQLu8oxCagwc\nUtVdGfXhIXf7SFXjRaQPMAU3cmGMqq4Ukf7AAlWdBIzGNTGjcC2EzoGL+MJ5ec1vAfmAbzx96ltV\ntX3Agr5AXl5zSPHymqcAbUVkFZAAPKWq+wMX9YXx8pqfAD4Skcdwt1G6BfMveSLyBe72X1FPP8nL\nQHYAVR2B6ze5FogCjgP3ZOj5g/jvzhhjTAYLxdtHxhhjzpMlBWOMMUksKRhjjEliScEYY0wSSwrG\nGGOSWFIwmZaIJIjIkmSPiDSOjUitqqS/iUikiAz1bLcUkabJXuslInf5MZa6wV411PhXyM1TMCHl\nhKrWDXQQ58ozQe70JLmWwFFglue1ERl9PhEJ99TwSkldXFmTyRl9XhOarKVggoqnRfC3iCzyPJqm\ncEwNEZnnaV0sE5FKnv1dku0fKSJhKbx3s4gM8hw3T0QqevaXE7cOxen1KMp69t8mIitEZKmIzPDs\naykiP3taNr2AxzznvEJE+onIkyJSTUTmnXVdyzzb9UXkLxFZKCJTUqqAKSJjRWSwiPwJDBKRhiIy\nS9yaArNEpIpnBnB/oJPn/J1EJK+4ev3zPcemVFnWZGWBrh1uD3uk9sDNyF3ieUz07MsD5PJsV8LN\nagWIwFN/HngfuNOznQPIDVQDfgKye/YPB+5K4Zybgb6e7buAnz3bPwF3e7bvBX7wbC8HSnm2C3r+\nbJnsff2AJ5N9ftJzz3VV8Gw/A7yAm7k6Cyjm2d8JN4v37DjHAj8DYZ7nFwHhnu02wHee7W7AB8ne\nNxDocjpeYB2QN9A/a3tknofdPjKZWUq3j7IDH4hIXVzSqJzC+2YDfUWkNPC9qq4XkdZAfWC+p8xH\nbiC1GlBfJPtziGe7CXCzZ/sz3BoNADOBsSLyNfD9uVwcrohbR+AN3Jd/J6AKrpDfVE+cYUBqdW2+\nUdUEz3YBYJynVaR4yiKkoC3QXkSe9DzPBZQFVp9j7CZEWVIwweYxYA9QB3f78z+L56jqBBGZC1wH\nTBGR+3Dlhsep6nNenENT2f7PMaraS0Qaec61xJOsvPUVrhbV9+6jdL2I1AJWqmoTL95/LNn2q8Cf\nqnqT57bV9FTeI8Atqrr2HOI0WYj1KZhgUwDYpa5Wflfcb9L/IiIVgI2qOhRXUbI2MA24VUSKe44p\nLKmvU90p2Z+zPduzOFM48U7gH8/nXKqqc1X1JSCaf5c0BjiCK+P9H6q6AdfaeRGXIMCVui4mbl0A\nRCS7iNRIJc7kCgA7PNvd0jj/FOAh8TRDxFXPNSaJJQUTbIYDd4vIHNyto2MpHNMJWCEiS4CquKUL\nV+Hu2f/u6dCdCqS2hGFOT0vjEVzLBOBh4B7Pe7t6XgN4S0SWe4bDzsCtIZzcT8BNpzuaUzjXV0AX\nzqwHcApXzn2QiCzF9Tv8pzM9BW8Cr4vITP6dKP8Eqp/uaMa1KLIDyzwxv+rFZ5ssxKqkGpOMuAV5\nIlU1OtCxGBMI1lIwxhiTxFoKxhhjklhLwRhjTBJLCsYYY5JYUjDGGJPEkoIxxpgklhSMMcYksaRg\njDEmyf8BWO5K7mCpc0IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1ad8051c588>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_score, test_score, auc_roc, RF_model = RFmodel(n_estimators = 50,\n",
    "                                                     min_samples_leaf=120)\n",
    "\n",
    "print(\"Predicting on test data...\")\n",
    "print(\"The Random Forest model yields : \\nTrain score = {0}\\nTest score = {1}\\nAUC_ROC = {2}\".format(train_score,test_score, auc_roc))\n",
    "pred_RF_test_proba = RF_model.predict_proba(X_test)[:,1]\n",
    "fpr, tpr, _ = roc_curve(y_test, pred_RF_test_proba,pos_label=1)\n",
    "df = pd.DataFrame(dict(fpr=fpr, tpr=tpr))\n",
    "plt.plot(fpr,tpr)\n",
    "plt.plot([0,1],[0,1],'r--')\n",
    "plt.ylabel('True positive rate')\n",
    "plt.xlabel('False positive rate')\n",
    "print(\"roc_auc score: {0}\".format(roc_auc_score(y_test,pred_RF_test_proba)))\n",
    "\n",
    "print(\"Predicting on validation data ...\")\n",
    "pred_RF_valid = RF_model.predict(data_valid[features])\n",
    "print(\"This model predicts {0:.2f}% of positive results on the validation data\".format(100*pred_RF_valid.sum()/len(data_valid)))\n",
    "pred_RF_proba_valid = RF_model.predict_proba(data_valid[features])[:,1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pipelining a PCA and a Logit classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import decomposition\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting PCA ...\n",
      "Done\n",
      "Fittin estimator (pipeline)...\n",
      "Done !\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAADICAYAAADItVzhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJztnXl8VNXZx7/PhJBASAiQEAyERBbZ\nWkUNi1AFrcXlrQt9BReg2KqAVYva+qLii6h1a/W1tbVSEQHR4lYti0pBCsUNBJQdgqJAIBDCmrBk\nf94/7s0YwiQzN8lwZznfz2c+c++dc+753TvzzDnnuec8R1QVg8EQXXjcFmAwGE4/xvANhijEGL7B\nEIUYwzcYohBj+AZDFGIM32CIQozhGwxRiDF8gyEKMYZvMEQhTdwWECgpKSmalZXltgyDIWRZvXr1\nflVNDSRt2Bh+VlYWq1atcluGwRCyiMiOQNOapr4hYnjlk+945ZPv3JYRFhjDN0QMn23bz2fb9rst\nIywIm6a+weCPl0f3cVtC2GBqfIMhComYGv+8xxZRVl7JmoeHEOORWtNVVlaya9cujh07dhrVGU4H\nRcVlACTGx7qsJHgkJCTQoUMHPJ6G1dlBNXwRiQeWAXF2We+o6sMicibwBtAa+BIYpaqlDSmr8EQZ\n5ZVKRaXWafj79+9HROjWrVuDb54htNhxwPozz2yT4LKS4FBZWcnu3bvZv38/bdu2bdC5gv3LLwEu\nUdVzgN7A5SLSH3gaeE5VuwKHgFsaWpDHNvZKPxGFDh8+TFpamjH6CCSzTULEGj2Ax+MhLS2NI0eO\nNPxcjaCnVtTiqL0ba78UuAR4xz4+E7i2oWXFiGX4FZV1G35FRQWxsZHbFDRENrGxsZSXlzf4PEGv\n9kQkRkTWAPuARcA24LCqVqnfBbSvJe8YEVklIqsKCgrqLCcmwBrfPm/A+g3hw76iYvYVFbstI6g0\n1m836IavqhWq2hvoAPQFevhKVkvel1Q1W1WzU1PrHolYdT8qKxsk1xDGFJdWUFxa0eDzDB48mJdf\nfrkRFIUup62jq6qHgaVAfyBZRKocix2AvIaev6rGrzBRg6OGpUuX0qFDB+9+xzYJdIzgPn5jElTD\nF5FUEUm2t5sBlwKbgSXAdXay0cCchpYVaB/fYDAEv8Y/A1giIuuAlcAiVZ0PTADuFZFvgDbAtIYW\nFKhXP5TJysrimWee4eyzz6Zly5Zcf/31FBf777POmTOH3r17k5SUROfOnVmwYAEAeXl5XH311bRu\n3ZouXbowdepUb57JkyczbNgwRo4cSWJiIj/84Q/ZunUrTz75JG3btiUjI4OFCxd60w8ePJgHHniA\nvn370rJlS6655hoOHjzo/Xzu3Ln06tWL5ORkBg8ezObNmwO+rvnz59O7d2+Sk5MZMGAA69at85v3\n2LFjXHHFFeTl5dGiRQtatGjB2pxv+XDJx2RnZ5OUlERaWhr33nuv4/sGsGPHDgYOHEhiYiJDhgxh\n//7vhwLXda1PP/007du3JzExkW7durF48WLAehT31FNP0blzZ9q0acPw4cO992/79u2ICDNnzqRj\nx46kpKTw+OOP+/3eG4SqhsXr/PPP17ro9/hHmjlhvu4+dLzOdJs2barzczfJzMzUPn366O7du/XA\ngQPavXt3ffHFF+vMs2LFCk1KStKFCxdqRUWF7tq1Szdv3qyqqhdddJHefvvteuLECf3qq680JSVF\nP/roI1VVffjhhzUuLk4XLFigZWVlOmrUKM3KytLf/e53Wlpaqi+99JJmZWV5yxk0aJCmp6fr+vXr\n9ejRo/qzn/1MR4wYoaqqOTk52rx5c124cKGWlpbq008/rZ07d9aSkhK/17V69WpNTU3V5cuXa3l5\nuc6YMUMzMzO1uLjYb94lS5Zo+/btvRp3HDim52b31VdffVVVVYuKivTzzz93fN8GDRqknTp10pyc\nHD1+/LgOGjRIJ0yY4Pdat2zZoh06dNDdu3erqup3332n33zzjaqqPvfcc9qvXz/Nzc3V4uJiHTNm\njN5www3edIDeeuutevz4cV2zZo02bdq01t9qbceBVRqgPblu0IG+/Bn+BU9Yhp978Fid6XzdtOFT\nPtO3Vu5UVdXS8godPuUzfffLXFVVPV5SrsOnfKZz11hf5pETpTp8ymf64fo8VVU9cLREh0/5TBdt\n3KuqqvmFJ3T4lM90yZZ8VVW/f0TVyczM1FmzZnn377vvPh07dmydecaMGaN33333Kcd37typHo9H\nCwsLvcfuv/9+HT16tKpahn/ppZd6P5s7d64mJCRoeXm5qqoWFhYqoIcOHVJVPenHr6q6ceNGjY2N\n1fLycn300Ud12LBh3s8qKio0PT1dlyxZ4ve6xo0bpw899NBJ2s866yxdunSp37w1DV9V9cILL9RJ\nkyZpQUFBXbet1vtWda2PPfaYd/+FF17Qyy67TFW1zmv9+uuvNTU1VRctWqSlpaUnnbN79+7eP11V\n1by8PG3SpImWlZV5DT83N9f7eZ8+fXT27Nk+9TWG4UfMKBZvUz/Mvfrt2rXzbjdv3pyjR4/WkRpy\nc3Pp3LnzKcfz8vJo3bo1iYmJ3mOZmZns3r3bu5+WlubdbtasGSkpKcTExHj3gZPKz8jIOOlcZWVl\n7N+/n7y8PDIzM72feTweMjIyTiqrtuvasWMHzz77LMnJyd5Xbm4ueXl5fvP6Ytq0aWzdupXu3bvT\np08f5s+f7zNdbffNX5l1XWuXLl344x//yOTJk2nbti033HCD9zp27NjB0KFDvdfYo0cPYmJiyM/P\nr9d1NpSIGavfEK/+m2Mv8G7HxnhO2m/WNOak/aT42JP2Wyc0PWm/bWL8Sfvpyc0c63FCRkYG27Zt\nO+V4eno6Bw8epKioyGv8O3fupH17n0MmAiI3N9e7vXPnTmJjY0lJSSE9PZ3169d7P1NVcnNzAyor\nIyODiRMnMnHiRMd6aj7T3nukmMS2GcyePZvKykreffddrrvuOg4cOEBCwsne/trumz/8XetNN93E\nTTfdRGFhIWPHjmXChAnMmjWLjIwMXnnlFQYOHHjKObdv3+5YR0OJmBo/Wr36t9xyC9OnT2fx4sXe\nsdxbtmwhIyODAQMG8MADD1BcXMy6deuYNm0aI0aMqHdZr732Gps2beL48eNMmjSJ6667jpiYGIYP\nH87777/P4sWLKSsr49lnnyUuLo4BAwb4Pedtt93GlClTWLFiBarKsWPHeP/99ykqKvKbNy0tjQMH\nDniHsJZVVPLm7NcpKCjA4/GQnJwM4G3FVKe2++aPuq41JyeHf//735SUlBAfH0+zZs28ZY8bN46J\nEyeyY4cVJKegoIA5cxr8MKveRIzhR4JXvz707duX6dOnc88999CyZUsGDRrk/XHNnj2b7du3k56e\nztChQ3nkkUf4yU9+Uu+yRo0axc0330y7du0oLi7m+eefB6Bbt2689tpr3HXXXaSkpDBv3jzmzZtH\n06ZN/Z4zOzubqVOncuedd9KqVSu6dOnCjBkzAtLTvXt3brzxRjp16kRycjIxxYdZ+ckSevXqRYsW\nLRg/fjxvvPEG8fHxp+St677VRV3XWlJSwv33309KSgrt2rVj3759PPHEEwCMHz+eq6++miFDhpCY\nmEj//v1ZsWJFQNcZDETDxFCys7O1rph7lz23jJz8IhbcfSHd2yXVmm7z5s306OFr8KChLgYPHszI\nkSO59dZb3ZYS9dT2GxaR1aqaHcg5IqbGr+ruRVtT3/A9e46cYM+RE27LCAsixvBjIsSr74snnnjC\nO0il+uuKK65wW1pIUWHHYzD4x3j1w4AHH3yQBx980FUNS5cudbX8QOjQqrnbEsKGiKnxPVHq1TcY\n6kOjGb6I9Gqsc9UHJ/Pxw8WhaXDGnsMn2HM4svv4jfXbbcwaf1YjnssxHu98/LpvTExMDGVlZadB\nkeF0U6nWK5IpKyujSZOG99Ab0/BdDWvjber7+UdMTk4mPz+fykj0AkY57Vs1o32r4I6UdJPKykry\n8/Np2bJlg8/VmM69UyxORDKAV4F2QCXwkqr+SUQmA7cBVfG0HlTVDxpSeKBe/ZSUFHbt2kVOTk5D\nijMYXCEhIYGUlJQGnyfYXv1y4Deq+qWIJAKrRWSR/dlzqvpMYxUUqFff4/HQsWPHxirWEEI8Mm8j\nAA9f5aq7KSxoTMM/JS6+qu4B9tjbRSKymVoCazaUqqa+vz6+wWBwYPhiTYUaAXRS1UdFpCPQTlW/\nAFDV/n7yZwHnAiuAgcCdIvJzYBVWq+CQjzxjgDGA31raiVffEJmYmj5wnDj3/gpcANxo7xcBLwSS\nUURaAP8A7lbVQuBFoDPWIht7gGd95VMHUXY9ZsiuwRAwTgy/n6reARQD2DW03+lXIhKLZfSvq+q7\ndt58tcJuVwJTscJuNwhvU9/U+FHL//5zA//7zw1uywgLnBh+mYjEYHvvRSQVy1NfK3b3YBqwWVX/\nr9rxM6olGwo0+NvyOvfMU7qoJT7WQ3xsxAxGDSpOnHvPA+8BbUXkcazw2A/5yTMQGAWst1fTAXgQ\nuFFEemP9iWwHxjoR7QtPBI/VNwTGxP/q6baEsCFgw1fV10VkNfBjrME616rqZj95PsH3wJ4GPbP3\nRYzx6hsMAePEq98f2KiqL9j7iSLST1XdCyNSDe+QXVPjRy0PvGvF43/yZ2e7rCT0cdIhehGoHvbz\nmH0sJPA29U2NH7UkN29KcnP/4b4Mzvr4otWmBqlqZbX171wnxnj1o54Jl3d3W0LY4KTG/1ZEfi0i\nsfZrPPBtsIQ5xXj1DYbAcWL444ABwG6sNe37YY+qCwWMV9/w27fX8tu317otIyxw4tXfB9wQRC0N\noqqpb4JsRC/pLU8No23wjROvfirWVNqs6vlU9ZeNL8s5Zsiu4d4h3dyWEDY4cc7NAT4GPgIqgiOn\n/hivvsEQOE4Mv7mqTgiakgZivPqGu9/4CoA/3nCuy0pCHyfOvfkicmXQlDQQ49U3dEptQafUFm7L\nCAuc1PjjgQdFpAQowxqKq6pa+3pVp5FoXTvP8D2//nFXtyWEDU68+on+U7lHoFF2DQaDw9BbItIK\n6Ap4n5uo6rLGFlUfYgKMsmuIXO78+5cA/OWm81xWEvo4eZx3K1ZzvwOwBugPfA5cUkee2qLstgbe\nxHo0uB0Y7iv0lhO8TX1T40ctPdNDotcZFjhx7o0H+gA7VPVirPh5BXVn8UbZ7YH1R3GHiPQE7gcW\nq2pXYLG93yBMjW/41eAu/GpwF7dlhAVODL9YVYsBRCROVbcAdY6YUNU9qvqlvV0EVEXZvQaYaSeb\nCVzrVHhNPMarbzAEjJM+/i4RSQb+CSwSkUNAXqCZa0TZTbNDb6Oqe0SkrQMdPjFRdg3jZq0GYMqo\n811WEvo48eoPtTcni8gSoCWwIJC8NaPsigS22paT8NrGq284LzPZbQlhg1/DF5Ek21hbVzu83n5v\nARz0k/+UKLtAvoicYdf2ZwD7fOVV1ZeAlwCys7PrtOhA184zRC5jLurstoSwIZAa/+/AT4HVWMEx\npcZ7p9oy1hZlF5gLjAaest/n1Ed8dWKMV99gCBi/hq+qP7UNeJCq7nR4/tqi7D4FvCUitwA7gWEO\nz3sKga6dZ4hcbp25EoCXR/dxWUnoE1AfX1VVRN4DHHlN6oiyC1a03kbD29Q3Xv2oZUDnhq8iGy04\n8eovF5E+qroyaGoagMcE4oh6fvmjM92WEDY4MfyLgbEisgMrwm7VJJ2QiGUcY49IMPPxDQb/ODH8\nK4KmohEwXn3D6Fe+AGDmLxu8FGPE4+Q5/g4Ae7BNyAU3M159w6U9GjwOLGpwMknnaqzlrNOxnrtn\nYg3BDYlFyb/36rssxOAaoy7IcltC2OBkrP5jWBNttqrqmVhe+U+DoqoeeMzaeQZDwDhaJltVDwAe\nEfGo6hKgd5B0OcZjYu5FPSNeXs6Il5e7LSMscOLcO2yPuV8GvC4i+7Cm3YYExqtv+OnZ6W5LCBuc\nGP41wAngHmAE1iSdR4Mhqj6YGt9wY9+6J3IZvseJ4Y8B3lbVXXw/lz5kiDFx9Q2GgHHSx08C/iUi\nH4vIHSKSFixR9cFjvPpRz/V/+5zr//a52zLCAifP8R8BHhGRs4Hrgf+IyC5VvTRo6hxghuwarju/\ng9sSwob6rG+/D9gLHABCZsSEN+aeaepHLcOyM9yWEDYE3NQXkdtFZClWcMwU4LZQGacP4DFe/ain\nrKKSMjM9MyCc9PEzsUJn9VLVh1V1U/UP7Zj71Dj2iojsE5EN1Y5NFpHdIrLGfjXKslxm7TzDyJdX\nMPLlFW7LCAuc9PH9hcBeDNRcyWAG8Bes2PrVeU5Vnwm07EAwXn3DDX1NUz9Q6tPHr41TAm6o6jI7\num7QMV59w9BzjXMvUJw09f3hxOTuFJF1dlfglC5CFSIyRkRWiciqgoK61+4wXn3DidIKTpRWuC0j\nLGhMww+UF4HOWOP892DN+POJqr6kqtmqmp2amlrnSY1X33Dz9C+4efoXbssIC4La1PeFquZ7M4hM\nBeY3RuHGq28Y2T/TbQlhQyBx9VvX9bmqVsXVDyh4ZlU8fXt3KLChrvSBYlbSMVx1jpmkEyiB1PjV\n4+l3BA7Z28lYobHPhJP+ALyIyGxgMJAiIruAh4HBItLbPud2YGxDLwJMU98AhcVlACTFx7qsJPQJ\nJK7+mQAiMgWYq6of2PtXAHUO11XVG30cnlYPnX4Rr3MvGGc3hAO3zVwFwJtjL3BZSejjpI/fR1XH\nVe2o6oci8lgQNNULs6CG4RcDs9yWEDY4Mfz9IvIQ8BpWM30k1nj9kMA09Q2X/+AMtyWEDU4e590I\npALv2a9U+1hIENvEMnwzVjt6OXislIPHSt2WERY4GbJ7EBgvIi1U9WgQNdWLZrExABSXGcOPVm5/\nbTVg+viB4CS89gDgZaylsTuKyDnAWFX9VbDEOSHeNnwzcit6ue3CWhduNtTASR//OeAyrCWuUdW1\nInJRUFTVg7gmHkSgtKKS8opKmsS4MSjR4CaX9gypoFAhjSPrUNXcGodCpnoVke+b++WmuR+N7Csq\nZl9RsdsywgInhp9rN/dVRJqKyG+xVtIJGZo3Nc39aOauv3/FXX//ym0ZYYGTpv444E9Ae2AXsBC4\nIxii6ku818FnDD8auX1wZ7clhA1OvPr7seLphyxVTf0TxvCjksHdQiYEZMjjxKufCtwGZFXPp6q/\nbHxZ9aOZ3dQ/bpr6UUne4RMApCc3c1lJ6OOkqT8H+Bj4iBBy6lXHPNKLbu55cw1gnuMHghPDb66q\nE4KmpBGocu6ZPn50ctclXd2WEDY48erPdxoRt5You61FZJGIfG2/1xp6yymmjx/d/KhrCj/qmuK2\njLDAieGPxzL+EyJSKCJFIlLoJ88M4PIax+4HFqtqV6zIvP6i9wZMM9PUj2p2HjjOzgPH3ZYRFgRs\n+KqaqKoeVW2mqkn2fpKfPMuAmgE6ruH7RTdnAtc6UlwH8VXOPVPjRyX3vbOW+95Z67aMsCCQ0Fvd\nVXWLiNSMmQ+Aqn7psMy0qtBbqrpHRBrtGUzzquf4psaPSu75yVluSwgbAnHu3Yu1RLavaLgKXNKo\niqohImPssunY0f/a51WP80wfPzrp36mN2xLChkBCb42x3y9upDLzqwJuisgZWItw1lb2S8BLANnZ\n2X4jbMQb515Us63Ami3eObWFy0pCH0fhtUXkB0BPIL7qmKrWXB7LH3OB0cBT9vsch/lrxTj3opsH\n310PmOf4geBk5N7DWBFzewIfAFcAn3DqunjV8/iKsvsU8JaI3IIVpXdYPbWfQjPzHD+q+Z/Lu7kt\nIWxwUuNfB5wDfKWqvxCRNKzAHLVSS5RdCDAGv1OamyG7Uc35mXUuAWGohpPn+CdUtRIoF5EkrL55\nSIU8MX386CZnbxE5e4vclhEWOKnxV4lIMjAVa5GNo0BILVRW1cc/WlzushKDG0yaYw0QNX18/ziZ\nllsVW2+KiCwAklR1XXBk1Y8zUxIA2Ly3EFX1LrJhiA4evLKH2xLChkAG8PgcuFP1WT0G8ASNDq2a\n0TYxjn1FJXy7/5h5rBNlnJOR7LaEsCGQGr/WZawJ8gAep4gI52e24sMNe1m945Ax/ChjY94RAHql\nt3RZSegTyACexhq4c1rIzmrNhxv28od/5XBBpzZktG7utiTDaeLReZsA08cPhIC9+iISLyL3isi7\nIvIPEblbROL95zy9XN8ngzNTEigoKuHdL3e7LcdwGpl0VU8mXdXTbRlhgZPHea8CvYA/A3/BGsgz\nKxiiGkKLuCbexRP3FppQy9FEr/SWppkfIE4e53VT1XOq7S8RkZCcA5mWZDVE9hnDjyrW5h4GjJMv\nEJzU+F+JSP+qHRHpB3za+JIaTpXh55vFFaKKJz7YzBMfhNRSDyGLkxq/H/BzEdlp73cENovIekBV\n9exGV1dP0pLiAMgvLHFZieF08ug1P3BbQtjgxPBrhtAKWVJbxCEC+4+WmHX0oohu7RLdlhA2OLGI\nrqq6o/oLGFxtO2RoEuMhpUUcqlBw1NT60cLqHQdZvaNmpDeDL5wY/iQReVFEEkQkTUTmAVcFS1hD\nMc396OP3C3L4/YIct2WEBU6a+oOA3wBr7P1Jqjq7vgWLyHagCGtxjnJVza7vuXyRlhjPBgrJN579\nqOGJn/3QbQlhgxPDb4Xl4NsGdAAyRURU1W9IrDq42F6Tr9Fpax7pRR1miHbgOGnqLwc+VNXLgT5A\nOiH6OA9MUz8aWf7tAZZ/e8BtGWGBE8O/FCgTkUmqegJ4hoYthqHAQhFZbUfTPQURGSMiq0RkVUFB\ngaOTt7NrfDN6L3p4btFWnlu01W0ZYYGTpv4DQCXWbLxHsfrnz2LV/vVhoKrm2XH1F4nIFnsBDi9O\no+xWxzuIxxh+1PCH687xn8gAOBzAo6rnichXAKp6SESa1rdgVc2z3/eJyHtAX2BZ3bkCp63d1N9n\nmvpRQ8c2ZiZmoDhp6peJSAxWEx0RScVqATjGfiSYWLUNDAE21J3LGWbYbvTxydf7+eTroPiKIw4n\nNf7zwHtAWxF5HCvq7kP1LDcNeM8OjdUE+LuqLqjnuXzSunlTmniEw8fLKC6r8AbiNEQuf/731wBm\nxdwAcBJz73URWY0VGluAa1W1XjMiVPVbrFDdQcPjEdomxpF3pJh9hSWmGRgFPHd9b7clhA2OVtJR\n1S3AliBpaXSyUhLIO1LMpj2FxvCjgPTkZm5LCBsievZKdpa1wMKq7Wb8djSwNGcfS3NqXYrRUA1H\nNX640SerFQArdxxyWYnhdPDi0m0ADO7WaCuvRywRbfjndWxFjEfYsPsIRcVlJMbHui3JEET+fNO5\nbksIGyK6qZ8Q14TzOiZTUal8bB7zRDxtE+Npmxhy8V9Dkog2fIAf90gD4I2VuTRsPpEh1PloUz4f\nbcp3W0ZYEPGGf6lt+Mu2FjD8b5+z69BxlxUZgsXUj79l6sffui0jLIh4w+/StgW//++zSWkRx8rt\nhxg17QsOHy91W5YhCLw48nxeHHm+2zLCgog3fIDhfTJYdM9F9Dgjie/2H2PsrNUcLTEr6kYarROa\n0jqh3tNHooqoMHyAVglNmTY6m7aJcaz47iCXPvsfluTso7yiXtMNDCHIgg17WLBhj9sywoKoMXyw\nRna9dms/zunQkr2Fxfxi+kp+MWOlcfpFCNM/3c70T7e7LSMsiCrDBzgrLZF3bh/AXZd0AeDjr/ez\nzDzqiwimjs5m6uhGDd0YsUSd4QPExnj4zZBu3HdZNwCe/GCzic0XASTFx5JkBmkFhGuGLyKXi0iO\niHwjIg0J4VVvRvbPJLNNc7bsLWLoXz/j7VW5HDxmPP7hyry1ecxbm+e2jLDAlSG7dkCPF4CfALuA\nlSIyV1U3nU4dLZvF8o/bB3DrzFWsyT3Mfe+swyPQ44wk0pLiSWnRlKT4WBLimtAirgnN42KIbxJD\nfGwMzZp6SIyPJblZLC2bxdIivgnxTWLweOR0XoKhGq8tt9Z1ueqcdJeVhD7ihmNLRC4AJqvqZfb+\nAwCq+mRtebKzs3XVqlVB0VNcVsE/vtzFvzbm8/m2/ZRV1P+eNIuNISHO+nNo4hGaxHho4hFiPOLd\nr9qO8Qix9n6MWPsiICJ4BDxi7XtO2re2RUA49U9Gahzy9TckNRP5TOPjWI2z+U7j/1y+yj/lSADl\n1zx3WUUlgtWV86+zHtcSiG4fyQL5nnydq2aan56dTlZKgo8Sq9LL6kDXp3Brkk57ILfa/i6smP2u\nEB8bw4h+mYzol0lhcRnb9h1l/9FSCopKKCou41hJOUdLKjheWk5JeSXFZRWcKKug8EQZR06UceRE\nOUdLyiguq+SE/ZnB0Nj0TE+q0/Cd4Jbh+/qjPKWatcNujwHo2LFjsDUBloPo3I6t6pW3slI5UVbB\n8dIKTpRWUF5ZSUWlUl6plFfoSfsVlUpZxcmfK0qlgqqiCpVq7VeqotW2q9LUpOYhn2n85PGVprZz\n+SvfOpcGkMb5eXyl25h3BIBe6S2rpQngPp1anI80/s/j81wBfAeBltexdeMYPbhn+LuAjGr7HYBT\nvDINCa/tBh6PkBDXhIS4iJ7tHLJc/7fPAbjj4i4uKwl93PqFrgS6isiZwG7gBuAml7QYIoTXbnWt\ntxh2uGL4qlouIncC/wJigFdUdaMbWgyRQ02nnqF2XGuTquoHwAdulW+IPN5eZfmLh2Vn+ElpMH+R\nhojhndW7eGf1LrdlhAWuPMevDyJSAOzwkywFCJeB90ZrcIhmrZmqmhpIwrAx/EAQkVWBDmBwG6M1\nOBitgWGa+gZDFGIM32CIQiLN8F9yW4ADjNbgYLQGQET18Q0GQ2BEWo1vMBgCwBi+wRCFRIzhh0JE\nn7oQke0isl5E1ojIKvtYaxFZJCJf2+/1mxbYcG2viMg+EdlQ7ZhPbWLxvH2f14nIeSGgdbKI7Lbv\n7RoRubLaZw/YWnNE5LLTqDNDRJaIyGYR2Sgi4+3joXFf1Z7yGc4vrPH+24BOQFNgLdDTbV01NG4H\nUmoc+z1wv719P/C0S9ouAs4DNvjTBlwJfIg1tbo/sCIEtE4GfusjbU/7txAHnGn/RmJOk84zgPPs\n7URgq60nJO5rpNT4fYFvVPV8swakAAAExUlEQVRbVS0F3gCucVlTIFwDzLS3ZwLXuiFCVZcBB2sc\nrk3bNcCrarEcSBaRM06P0lq11sY1wBuqWqKq3wHfYP1Wgo6q7lHVL+3tImAzVgCakLivkWL4viL6\ntHdJS20osFBEVtsBRgDSVHUPWD8UIJQWdq9NW6je6zvtJvIr1bpMIaFVRLKAc4EVhMh9jRTDDyii\nj8sMVNXzgCuAO0TkIrcF1ZNQvNcvAp2B3sAe4Fn7uOtaRaQF8A/gblUtrCupj2NB0xophh9QRB83\nUdU8+30f8B5WkzO/qjlnv+9zT+Ep1KYt5O61quaraoWqVgJT+b4576pWEYnFMvrXVfVd+3BI3NdI\nMXxvRB8RaYoV0Weuy5q8iEiCiCRWbQNDgA1YGkfbyUYDc9xR6JPatM0Ffm57ofsDR6qarm5Roy88\nFOvegqX1BhGJs6M9dQW+OE2aBJgGbFbV/6v2UWjc19PpkQ2yF/VKLM/pNmCi23pqaOuE5V1eC2ys\n0ge0ARYDX9vvrV3SNxuriVyGVfPcUps2rCbpC/Z9Xg9kh4DWWbaWdVgGdEa19BNtrTnAFadR54+w\nmurrgDX268pQua9myK7BEIVESlPfYDA4wBi+wRCFGMM3GKIQY/gGQxRiDN8QFojIYBEZ4LaOSMEY\nviFcGAwYw28kjOGHESKSZU/znGpP9VwoIs1qSdtFRD4SkbUi8qWIdLYHh/xBRDbYU4Svt9MOFpH/\niMhbIrJVRJ4SkREi8oWdrrOdboaITBGRj+10P7WPx4vIdDvtVyJysX38ZhF5V0QW2NNQf19N3xAR\n+dzW9rY9tLVq+vIj9vH1ItLdHus+DrjHnnZ7oYgMs69jrYgsC+Z9j0jcGDBiXvUeFJIFlAO97f23\ngJG1pF0BDLW344HmwH8Di7CmMacBO7Gmjw4GDtvbcVjrGT5i5x0P/NHengEswKowumINoIkHfgNM\nt9N0t88bD9wMfAu0tPd3YA1LTQGWAQl2ngnAJHt7O3CXvf0r4GV7ezLVpt5iDXJpb28nu/3dhNvL\n1Pjhx3equsbeXo31Z3AS9vDg9qr6HoCqFqvqcazRZLPVGteeD/wH6GNnW6nWVNISrNFjC+3j62uU\n8ZaqVqrq11hG3d0+7yy7rC1YBn6WnX6xqh5R1WJgE5CJNd+8J/CpiKzBGrqaWa2MqnHtPq/P5lNg\nhojchvVHZnCAWc85/Ciptl0B+Grq+5rpVdfxmuetrLZfycm/k1OWs3dw3gr7XAIsUtUb/eSpSn8K\nqjpORPoB/wWsEZHeqnqgDh2GapgaPwJRa/rnLhG5FsCepNIcq3l9vYjEiEgqVjQbp5NWhomIx+73\nd8IaA78MGGGXdRbQ0T5eG8uBgSLSxc7T3M5XF0VYkWyw83RW1RWqOglrGSqzUqYDjOFHLqOAX4vI\nOuAzoB3WdOB1WJOF/g38j6rudXjeHKwuwofAOLsJ/1cgRkTWA28CN9tdBp+oagFW/3+2rW85Vpeh\nLuYBQ6uce8AfbOffBqw/nrUOryOqMZN0DAEjIjOA+ar6jttaDA3D1PgGQxRiavwwR0ReAAbWOPwn\nVZ3uhh5DeGAM32CIQkxT32CIQozhGwxRiDF8gyEKMYZvMEQhxvANhijEGL7BEIX8PzVniCD8OuX2\nAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1ad81d63780>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "logistic = LogisticRegression(max_iter=10)\n",
    "pca = decomposition.PCA()\n",
    "pipe = Pipeline(steps=[('pca', pca), ('logistic', logistic)])\n",
    "# Plot the PCA spectrum\n",
    "print(\"Fitting PCA ...\")\n",
    "pca.fit(X_train)\n",
    "print(\"Done\")\n",
    "plt.figure(1, figsize=(4, 3))\n",
    "plt.clf()\n",
    "plt.axes([.2, .2, .7, .7])\n",
    "plt.plot(pca.explained_variance_, linewidth=2)\n",
    "plt.axis('tight')\n",
    "plt.xlabel('n_components')\n",
    "plt.ylabel('explained_variance_')\n",
    "# Prediction\n",
    "n_components = [20, 100, 150]\n",
    "Cs = np.logspace(-4, 4, 3)\n",
    "# Parameters of pipelines can be set using ‘__’ separated parameter names:\n",
    "estimator = GridSearchCV(pipe,\n",
    "                         dict(pca__n_components=n_components,\n",
    "                              logistic__C=Cs))\n",
    "print(\"Fitting estimator (pipeline)...\")\n",
    "estimator.fit(X_train, y_train)\n",
    "print(\"Done !\")\n",
    "plt.axvline(estimator.best_estimator_.named_steps['pca'].n_components,\n",
    "            linestyle=':', label='n_components chosen')\n",
    "plt.legend(prop=dict(size=12))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_pipe_test = estimator.predict(X_test)\n",
    "pred_pipe_valid_proba = estimator.predict_proba(data_valid[features])\n",
    "pred_pipe_valid = estimator.predict(data_valid[features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The PCA/Logit pipeline predicts 10.26% of positive results on the test set and 10.27% on the validation set\n"
     ]
    }
   ],
   "source": [
    "print(\"The PCA/Logit pipeline predicts {0:.2f}% of positive results on the test set and {1:.2f}% on the validation set\".format(100*pred_pipe_test.sum()/len(X_test),100*pred_pipe_valid.sum()/len(data_valid)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GradientBoostingClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dealing with data imbalance\n",
    "\n",
    "I have decided to oversample the least frequent values (\"1\") instead of downsampling the most frequent ones (\"0\")."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 3.64% of positive values in the original data set\n",
      "There are 7.04% of positive values in the train set\n"
     ]
    }
   ],
   "source": [
    "print(\"There are {:.2f}% of positive values in the original data set\".format(100*data['target'].sum()/len(data)))\n",
    "\n",
    "#Fetch only positive entries\n",
    "df_positives_train_data = train.loc[train['target']==1]\n",
    "\n",
    "#Duplicate to balance data\n",
    "train = train.append(df_positives_train_data)\n",
    "print(\"There are {:.2f}% of positive values in the train set\".format(100*train['target'].sum()/len(train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 7.04% of positive values in the train set\n",
      "There are 3.64% of positive values in the test set\n"
     ]
    }
   ],
   "source": [
    "X_train, y_train, X_test, y_test = train[features], train[target], test[features], test[target]\n",
    "print(\"There are {:.2f}% of positive values in the train set\".format(100*y_train.sum()/len(y_train)))\n",
    "print(\"There are {:.2f}% of positive values in the test set\".format(100*y_test.sum()/len(y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "roc_auc score: 0.6261686138419089\n",
      "Predicting on validation data ...\n",
      "This model predicts 0.00% of positive results on the validation data\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xd0VNUWwOHfJlSRIh3pVToKkSpF\nQUQsiL0hWEDwWQAb1odYsTcQUBBERUTgiYggCApSpPcaQgsthN5J2e+PM8QYU4Yk05L9rTWLO3fu\nzOwrmJ3T9hFVxRhjjAHIFegAjDHGBA9LCsYYYxJZUjDGGJPIkoIxxphElhSMMcYksqRgjDEmkSUF\nY4wxiSwpGGOMSWRJwRhjTKLcgQ7gfJUoUUIrV64c6DCMMSakLF26NEZVS6Z3XcglhcqVK7NkyZJA\nh2GMMSFFRLZ7c511HxljjElkScEYY0wiSwrGGGMSWVIwxhiTyJKCMcaYRD5LCiIyUkSiRWRNKq+L\niHwsIhEiskpEGvkqFmOMMd7xZUthFNAxjdevBWp4Hj2Bz3wYizHGGC/4bJ2Cqs4RkcppXNIZ+Erd\nfqALRaSoiJRV1T2+iskYY0LJmbh4lu84zNZt+4jdF03DKy6lYYWiPv3OQC5eKwfsTPI8ynPuX0lB\nRHriWhNUrFjRL8EZY4y/nY6NZ/ySnUxavov9x8+w8+Apmm9fyVvTPuFYvoKsmDgjWycFSeGcpnSh\nqg4HhgOEh4eneI0xxoSiE2fi+HHFbmas28vczTHEJShFCuShYq6zfLpyNA2n/cCZylUp/flw6rWo\n4vN4ApkUooAKSZ6XB3YHKBZjjPGbXYdPMWtDNBOXRbF8x+HE882qFuOuJhXpXL8M1K8PGzfCM8+Q\nb8AAKFDAL7EFMilMBh4Vke+ApsARG08wxmRHp2PjiT56hlW7DvPprAg27D2W+Frpwvno274mN11W\njvxHD0OxYiACr78OFSpAeLhfY/VZUhCRsUBboISIRAH/BfIAqOpQYCrQCYgATgL3+yoWY4zxt4Mn\nzrJo6wGmrdnL/1b8sxOkWsmCPN6uBjc2vBgRAVX45ht44gl46y3o0QO6dAlI3L6cfXRXOq8r8B9f\nfb8xxvhTQoKycd8xZq7bx+dzIzl6Oi7xtXJFC3B9g7K0vaQUTaoUIyxXkiHVnTuhVy+YOhWaNYOW\nLQMQ/d9CrnS2McYEi4QEZfnOQ3wxdyuzN0ZzOjYBgGIF89K6ZklubVye1jVKUPSCvCl/wNix8PDD\nEB8PH34Ijz4KYWF+vIN/s6RgjDHn4eTZOAbPjuDPzTGsjDqSeL504Xw836k6jSpeRL1yRbz7sIsu\ngqZNYfhwqOL7mUXesKRgjDFpOB0bz9TVe1i87SAz1kUTc/xM4mvNqxanzsWFuS28PLXKFE7/w+Li\n4IMP4OxZeOEF6NgRrrnGDSwHCUsKxhiTRHyCsmDLAQbPjmDTvmMcOHH2H69fVrEoHeuW4eE21c7v\ng1euhAcfhKVL4fbb3eCySFAlBLCkYIwxgOsWenPqBqas2s2hk7GJ57tcVo5r6pahVY0SFMyXgR+Z\nZ87Aa6+5WUXFisH48XDLLUGXDM6xpGCMydGWbDvIsDmRzFi3L/HcqzfV44rqJahSomDmv2DzZhg0\nCO6+G95/H4oXz/xn+pAlBWNMjrP3yGnenraBhZEH2H3kdOL5Hq2q8Hyn2m7tQGYcPw4//gj33AP1\n6sGGDVC1aiaj9g9LCsaYHGH5jkP8uGI38yJi2Bx9PPF89xaV6dWmGmWK5M+aL5oxA3r2hO3boVEj\nqF07ZBICWFIwxmRTqsqirQcZvWAba3cfZfuBkwBckDeM+uWK8Ejbalxbv2zWfeGhQ/DUUzByJNSs\nCX/84RJCiLGkYIzJdn5YGsWHMzcRdehU4rmr65Tm9S71KFUoi1oEScXHu5XImzbBc8/Byy9Dfh98\njx9YUjDGZAuqyryIA7z760ZW7HSVR+9qUpFnrrmEiwqmsqI4s2Ji3IyisDB44w2oWNF1GYUwSwrG\nmJCmqsxcH80nszazyrPCuFP9Mrx9a0MuzMgUUu++FMaMgT593FTTnj3hppt8811+ZknBGBOSYuMT\nmLp6D+9M35jYTfRwm6r0al3Ndy0DcAPIDz8M06dDixbQurXvvisALCkYY0LGsdOxfPzbZmas28c2\nz8AxuJbBe7ddSoG8Pi4m9/XX0Lu3ayl88gk88gjkyuXb7/QzSwrGmKAXF5/Ad4t38uL/1iSeq1ay\nIE2qFOe/N9Qhfx4/VRYtWdINKA8bBpUq+ec7/cySgjEmaK3fc5S3p21g2Y7DHDnlSk88d22t8687\nlFGxsfDee+7Pl15yxes6dAjaEhVZwZKCMSZoqCq/b9zPkN8j2Hf0DDsOui6iqiULMrBzXTrVL0ue\nMD911yxf7grYLV8Od94ZtAXsspolBWNMwMXFJ/D53K189ntE4o5lFYoVoHuLylxTtwzNq/mxXtDp\n0zBwILz9NpQoARMmwM03++/7A8ySgjEmYI6fiWPI7Ag+nxtJbLwCruxEtxaVs6YYXUZERMC778J9\n97muo4suCkwcAWJJwRjjV6rK5JW7Gb8kij8jYgAoWSgf9zWrxKNXVc98MbqMOH4cJk2Crl1dAbuN\nG4NmJzR/s6RgjPGL6GOnGTwrgtELtieeK1skP49cWZ17mlQkV64A9dVPn+4Wn+3cCeHhrl5RDk0I\nYEnBGONja3YdYeKyXXw5fyuqkDd3Lu5vWZlH2lanSIE8gQvswAHo1w+++gpq1YK5c0OygF1Ws6Rg\njPGJPzfH8NKPa9gacwKAVjVK8HDralxRo0SAI+PvAnYREW6v5BdfDNkCdlnNkoIxJkut2XWEwbMj\n+GXNXgBublSOvu1rUqHYBQGODNi/3+18FhbmdkOrVAkuvTTQUQUVSwrGmCwRF59Av+9XMnnlbgAq\nFb+A8b2a+6ZU9flShVGjXHfRW2+52kWdOwc6qqBkScEYkymnzsbz2R9b+GJuJCfPxlOvXGE+vvMy\nqpa8MNChOdu2uYHkGTOgVSu48spARxTULCkYYzJk0daDvDplHat3HUk817d9TZ5oXyOAUSUzZowr\nYCcCQ4a4FkI2K2CX1SwpGGPOy7aYEzwxbgUrPRvZXFe/LJdXvoiuzSsTFqhppakpXdqVth461G2A\nY9JlScEY45XI/cd599eNTF3tBpDLFsnPxEdaULZIgQBHlkRsrCtPER/vtsTs0ME9jNcsKRhj0nTo\nxFn6T1zF9LX7AOhyWTkevKIK9coVCXBkySxbBg88ACtXwt13/13AzpwXnyYFEekIfASEAV+o6lvJ\nXq8IjAaKeq7pr6pTfRmTMcZ7f26O4YnvlnPgxFkAZj/VNnA1iVJz6hS88oqrV1SypCtXkU22xgwE\nnyUFEQkDBgNXA1HAYhGZrKrrklz2IvC9qn4mInWAqUBlX8VkjEnf6dh4vlqwja8WbE/c5nJ418Z0\nqFsmsIGlJjIS3n8funeHd97JcQXsspovWwpNgAhVjQQQke+AzkDSpKBAYc9xEWC3D+MxxqThz80x\n/LRyN+OW7ATggrxhdL70Yl65sS5FL/DhnscZcfQoTJzoEkHdurB5c7bdCc3ffJkUygE7kzyPApom\nu2YA8KuIPAYUBNr7MB5jTApOx8YzcMo6vv1rBwAtqxfnsgoX8WSHmoGpWJqeqVOhVy/YtQuaNnX1\niiwhZBlfJoWU/jVpsud3AaNU9T0RaQ6MEZF6qprwjw8S6Qn0BKho08qMyTITlkbx5PiVANzVpAJ9\nr64ZHCuQUxITA337wtdfQ506MG+eFbDzAV8mhSigQpLn5fl399CDQEcAVV0gIvmBEkB00otUdTgw\nHCA8PDx5YjHGnKddh08xZHYE33haB6/eVI+uzYL4t+1zBewiI91U0+efh3z5Ah1VtuTLpLAYqCEi\nVYBdwJ3A3cmu2QG0A0aJSG0gP7DfhzEZk6MdORXL0+NX8us6N720Xa1SfHzXZRTMF6Sz0/ftczOK\nwsLc7KJKlaBBg0BHla357F+CqsaJyKPAdNx005GqulZEBgJLVHUy8CTwuYj0xXUtdVdVawkYk8W2\nxpygT5JVyHlz5+KHXs1pUL5ogCNLhSqMHAlPPukK2PXqBTfcEOiocgSf/nrgWXMwNdm5l5McrwNa\n+jIGY3Ky42fi+Oz3CD6fu5WzcQlcVrEoD11RlesalA10aKmLjIQePWDWLGjTBtrb/BN/CtI2ozEm\nM46cjOXL+Vv5cOZmwO2BPK5ns+CpXJqa0aPhkUdcd9HQoS45WAE7v7KkYEw2s3T7Qe79YhGnYuMp\ncWFe+l5dk7ubVAzO6aXJXXwxXHUVfPYZlC8f6GhyJEsKxmQTK3Ye5qnxK4mIPu4qRd/TiE71g7ib\nCODsWTdmkJAAAwbA1Ve7hwkYSwrGhLizcQkMnLKWcYt3Ehuv1C9XhKFdG1OuaBBVL03J4sWugN2a\nNdC1qxWwCxKWFIwJUQdPnGXS8l2MmBvJ7iOnAZjWpxW1yhRO550BdvKkW2vwwQdQtixMnmwzi4KI\nJQVjQsymfcf4cOamxH0NAN65tQE3NyoffJvcpGTrVvjkEzeIPGgQFAmyEtw5nCUFY0JEXHwCr/28\nnlHztwHQqX4ZmlcrwV2XVyB3WJDP0DlyxBWwu/9+V8AuIgIqVEj/fcbvLCkYEwLmb4nhxUlriIw5\nQYc6pXn22lpUC/bppef8/LPbG3nPHmjeHGrVsoQQxCwpGBPE5kXE0P3LRcTGu4X+g26pzx2Xh0hR\nyP37oU8f+PZbqFfPtRRq1Qp0VCYdlhSMCUL7j53hpf+tYdpaN27QvnZp3uhSj1KFg7SCaXLx8XDF\nFW784JVXoH9/yBtkezKYFFlSMCZIxCcoU1fv4Ys/tybWKKp7cWGG3NOISsWDbAvM1OzdC6VKuRXJ\n770HlSu7VoIJGZYUjAkCy3ccov+E1Wzcd4xC+XPTsHwRHm9Xg3a1Swc6NO8kJMDnn8PTT7sZRb17\nw/XXBzoqkwHpJgURKQD0ASqpai8RqQ7UUNVffB6dMdnc9gMneGHSGv6MiAHgvuaVePn6OsE/myip\niAg3vfT3312JimuuCXREJhO8aSmMBFYDV3ie7wbGA5YUjMmg6GOneWvqBiYu3wVAqxoleLZjLeqV\nC7E5+19+6QrY5c3rWgoPPmirkkOcN0mhhqreJSK3AajqSQmJylrGBJ+tMSd4fuJqFkQeAOCyikXp\n074mbWqWDHBkGVSxomsZDB4M5coFOhqTBbxJCmc922QqgGcntbM+jcqYbObA8TN8MHNTYn0igBHd\nwkNnzOCcM2fgzTfdGMLAgdCunXuYbMObpPAqMA0oLyKjgTbAQz6NyphsYs2uIzw3cTWrdx1JPPdr\n39bULF0ogFFl0F9/ue6htWuhWzcrYJdNpZsUVPUXEVkCtAAEeFpVo30emTEh7ExcPO/P2MSwPyIB\nuKVRebpcVo6W1YuHxr4GSZ04AS+9BB9+6LqIpkyB664LdFTGR7yZffSrqnYAfkzhnDEmmZ0HT/L8\npNXM3RxDyUL5GH1/E+pcHOSVS9OyfTsMGeL2SX7rLSgcwvdi0pVqUhCRvEB+oLSIFMK1EgAKAyGy\nzt4Y/xq/ZCfPTFiFKlxXvyyf3n1Z6LUMAA4fhh9+gIcegjp13LRT2wktR0irpfAfoB9QCljL30nh\nKDDUx3EZEzJUlXGLdzL49wh2HjwFwJfdL+fKWqUCHFkG/fijW3wWHe1KVdSqZQkhB0k1KajqB8AH\nItJHVT/0Y0zGhIzF2w7S++ulxBx3E/LaXlKSwXc3omC+ECwWEB0Njz8O48ZBgwZu8xsrYJfjeDPQ\n/KGI1ALq4LqTzp3/1peBGRPMYuMT6D9hNROWRQHQolpxRnS7nAJ5wwIcWQbFx0PLlrBjB7z2Gjzz\nDOTJE+ioTAB4M9D8ItABqAVMB64B/gQsKZgcKfroafpPXM2sDdG0qFacV26sS41QnGIKsHs3lCnj\nCth99JErYFenTqCjMgHkTYGVO4ArgT2q2hVoiBXSMznQ2bgEhvwewRVvz2bWhmj6XV2Tb3s0C82E\nkJAAn33muoeGeoYIO3WyhGC8+uF+SlXjRSTOMwtpL1DVx3EZE1Qi9x+nz7gVrIo6QtWSBXnn1gY0\nrlQs0GFlzKZNroDdnDnQvj1ce22gIzJBxJuksFxEiuIK4y3BzT5a5tOojAkSK3ce5pNZm5m53q3X\nfL1LPe5pWinAUWXCiBHw6KOQPz+MHAndu9uqZPMPaSYFT+G7Aap6GBgsItOBwqpqScFka6rKiD+3\n8trP6wGoXbYww7s2pkKxCwIcWSZVruxaBoMHQ9mygY7GBKE0k4KqqohMARp7nkf4JSpjAuiPTfsZ\n+NNatuw/QaH8uRnbo1nolbQ+58wZePVVd/zaa1bAzqTLm+6jRSLSyFoHJrs7G5fA4NkRfPTbZooV\nzEvf9jXp3bYaeXOH0IY3Sc2f7wrYbdgADzxgBeyMV7xJClcAPURkC3ACt7JZVbWRTyMzxo9OnInj\n9mELWLv7KEUK5GFan1aUKpQ//TcGo+PH4YUX4JNPoEIFmDbNdkMzXvMmKdyU0Q8XkY7AR0AY8IWq\nvpXCNbcDA3D7NaxU1bsz+n3GZMTCyAM8O2EV2w+cpGfrqjzbsRZhuUL4N+odO2DYMPjPf+CNN6BQ\nCE6ZNQHjzYrmLRn5YBEJAwYDVwNRwGIRmayq65JcUwN4DmipqodEJESLxZhQlJCgfDBzE5/MckNl\ng26pzx2Xh2itx0OHYPx46NnTrTWIjISLLw50VCYE+XIRWhMgQlUjAUTkO6AzsC7JNT2Awap6CMD2\naTD+NHDKOkbN30btsoUZ0S2ci4sWCHRIGTNpktsnef9+aNMGLrnEEoLJMF+OoJUDdiZ5HuU5l1RN\noKaIzBORhZ7upn8RkZ4iskREluzfv99H4ZqcICFBmbA0ik4fzWXU/G00q1qMnx+7IjQTwt69cNtt\ncPPNrlTFokUuIRiTCV61FESkPFBDVWeLSD4gt6qeSO9tKZzTFL6/BtAWKA/MFZF6nnURf79JdTgw\nHCA8PDz5ZxiTrlNn4xk+J5IPZm5KPPdEuxo8dlV1coXi+EF8PLRqBTt3unGDp56yAnYmS3hTEO8B\n4FGgCFANqAQMAdqn89YooEKS5+WB3Slcs1BVY4GtIrIRlyQWexW9MV4Ys2AbL/24FoBcAg+0rMLT\nHS8hX+4QrGgaFeW6hsLC4OOPoUoVK29tspQ33UePA81w5S1Q1U24jXfSsxioISJVPLu43QlMTnbN\n/3DF9hCRErjupEjvQjcmdarKml1HuPHTPxMTQt/2NdnyRidevL5O6CWEhAQ3xbRWLVfIDtzKZEsI\nJot50310WlXPnttS0DOrKN32tqrGicijuHLbYcBIVV0rIgOBJao62fNaBxFZB8QDT6vqgQzeizGo\nKhOX7eKDmZuIOuR2QWtdsySD776MQvlDtHtlwwa3Lea8eW69wfXXBzoik415kxTmicgzQH4RuRK3\nTecUbz5cVacCU5OdeznJseK2/OzndcTGpOLnVXt4a9r6xC0xuzWvxCNXVqd04RBdhAbwxReugN0F\nF8Do0dC1q61KNj7lTVJ4BugJbACewP12P8yXQRlzPo6ciuWFSauZsmoPAD1aVeGxdjUoHKotg6Sq\nVYMbboBPP4XSpQMdjckBvEkKnXCrkT/zdTDGnK8l2w7y+Njl7D5ymva1S/PaTfUoUySEWwanT8PA\nge74jTfgyivdwxg/8SYp3A58KiKzgO+Amaoa79uwjEnb9gMn+M+3y1iz6ygAb95cn7uahOhq5HPm\nzXMF7DZudGMIVsDOBIA3ZS66etYmXAc8AAwXkV9UtZfPozMmmR0HTvLshFUsiHTzETrWLcPTHS+h\nWskLAxxZJhw7Bs8/7/Y4qFQJpk+HDh0CHZXJobxavKaqZ0TkR+AUbibR7YAlBeM3x07H8u70jXzz\n1w7iEpT2tUvRp33N0N3nIKmoKDeg/Nhj8PrrcGEIJzgT8rxZvNYet8agPTAP+AqwSqbGLxISlIFT\n1vH1wu3EJbjF7FMeuyL0k8GBA/D999C7N9Su7QrY2U5oJgh401LohRtLeExVT/k4HmMSJSQoD45e\nzOyNrt7V5/eF0752KSSU+9lVYcIEV9b64EG46ipXr8gSggkS3owp3OqPQIxJanXUEd6atp55EQd4\noGUVXrq+dmgnA4A9e1wymDQJGjeGX3+1AnYm6KSaFETkD1VtIyKH+Gchu3M7rxXzeXQmx/lzcwz9\nvl9B9LEzANzWuHz2SAjnCtjt2gVvvw19+0JuX1auNyZj0vpXeW5ydAl/BGJytsMnzzJo2kbGLtoB\nQMlC+fjqgSbULls4wJFl0s6dUK6cK2A3eLArYFezZqCjMiZVqRbEU9UEz+EIVY1P+gBG+Cc8kxOM\nnr+NSwfOYOyiHZQrWoD5/a9i8QvtQzshxMe7KqZJC9hdc40lBBP0vGm/Nkj6xFMQ73LfhGNykiMn\nY3lu0iqmrt5L/jy5ePe2hlzfIBvsGLZ+vVuEtmCBq2R6ww2BjsgYr6U1pvAs0B8oJCIHz53GjS9Y\nS8FkyrIdh+j99VL2HT1D/XJF+K5nMwrmywZ97MOHu/UGhQrBmDFwzz22KtmElLT+L3wbeA94E5cc\nALASFyajVJXZG6P5YWkUU1fvpXD+3HzboyktqmWjYasaNaBLF9d1VMqbbUeMCS5pJYXqqrpZRMYA\ndc+dPDcLRFVX+Tg2k438unYv78/YxIa9xwBoX7sUL11fh0rFCwY4skw6dQoGDHCtgbfesgJ2JuSl\nlRT6Aw8Cg1N4TYHWPonIZCsHjp+h9zfLWLTV9UD2aV+D+1tWoUiBbFDWes4cV7hu82bo1csK2Jls\nIdWkoKoPev5s5b9wTHYRfew0I/7cyrA/3O6qlYpfwIy+bcib25sdYIPc0aPQv7+bVVS1Kvz2m1uZ\nbEw24E3to5uBGap6TET6A42A11V1pc+jMyHpl9V76P3NMsD94jzq/ia0qVkywFFlod27YdQo6NfP\n7X1QMMS7wIxJwpvpHgNUdaKItABuAN7H7bzWzKeRmZBz+ORZHh6zlL+2HuSiC/Lw6k31uK5+2dBf\njQwQE+MK2D3yiFt7sHWr7YRmsiVv2vLnZhtdDwxR1QlAPt+FZELRniOnuGPYQv7aepBbGpVn1pNt\nub7BxaGfEFRh3DioUwf69IFNm9x5Swgmm/KmpbBHRAYD1wKNRSQv3iUTkwOcjo3nhUlr+Hn1bk7H\nJvD2rQ24PbxCoMPKGrt3u9LWkydDeLgbO7AVySab83Y7zk7AJ6p6SEQuJsm6BZNzTVuzl3d/3UhE\n9HEK5cvNuP80p2GFooEOK2vEx0Pr1q6A3bvvwhNPWAE7kyN4Uzr7uIisA9qKSFtgrqr+4vPITNA6\nfPIsr05Zz4RlUQAM7FyX+5pXDmxQWWX7dihf3hWwGzLEzS6qXj3QURnjN+l2A4nIo8D3QEXP43sR\necTXgZngtPvwKa77+E8mLIuife1SrB/YMXskhPh4eP99twvauQJ2HTpYQjA5jjft4Z5AE1U9DiAi\nbwDzgSG+DMwEn+hjp7nx03nEHD/DG13qc3fTioEOKWusWeMK2C1aBNdfDzfdFOiIjAkYb5KCALFJ\nnsd6zpkcZFXUYe4buYjDJ2MZ2T2cq2plk9k3Q4fC449DkSLw7bdw5522KtnkaN4khTHAQhGZgEsG\nNwGjfRqVCRo7Dpxk3JIdfD5nK2fjExhwQ53skRDOlaSoXRtuuw0+/BBKZqMFdsZkkDcDzW+LyGzg\nXLmLXqq62LdhmWAwYWkUT453C9fz58nFt/c3pUX1EK9oevIkvPyyG0geNAjatHEPYwzgXUsB4Izn\nkeD502Rzv6zew1M/uIQwrGtjrqlbJsARZYHff3cF7LZscSuTrYCdMf/izeyjF4CxQFmgPPCtiDzn\n68BMYJyOjeep8Svp/c0yVOGv59uFfkI4cgQefvjvktazZrn9ki0hGPMv3qxMvhe4XFVfVNUXgCbA\nfd58uIh0FJGNIhLhKaaX2nW3ioiKSLh3YRtfmLluH+GvzeSHpW79wYy+rSldOH+Ao8oCe/bA11/D\nU0/BqlW234ExafCm+2h7sutyA5Hpvcmzl/Ng4GogClgsIpNVdV2y6woBjwN/eRu0yXpfzI3ktZ/X\nA/D2LQ24/fIQL1Wxfz98953bGrNWLdi2zQaSjfGCN0nhJLBWRKbjNtfpAPwpIu8DqGq/VN7XBIhQ\n1UgAEfkO6AysS3bdq7itP586//BNZp06G89jY5cxc300AHOfuZIKxS4IcFSZoApjx7pppkePwjXX\nuHpFlhCM8Yo3SeFnz+OchV5+djlgZ5LnUUDTpBeIyGVABVWdIiKWFPwsIUF5ftJqZq6Ppn3tUrx1\nSwNKXBjCBXB37nQF7H7+GZo2hREjrICdMefJmympIzL42SmN4mniiyK5gA+A7ul+kEhP3MpqKlbM\nJqtoA+zIqVieGr+SGev20apGCb7odnmgQ8qcuDho2xb27oUPPnDdRmFhgY7KmJDjy7KPUUDSjuny\nwO4kzwsB9YDfPTX3ywCTReRGVV2S9INUdTgwHCA8PFwxmfLNX9t5YdIaAJpULsZXDzQJcESZsG0b\nVKjgKpgOG+YK2FWtGuiojAlZvtwXYTFQQ0SqePZguBOYfO5FVT2iqiVUtbKqVsZ1S/0rIZisNWfT\nfgZMXgvA0Hsb832v5qG5EU5cnCtpXbu2q2YK0L69JQRjMsnrloKI5FNVrxeuqWqcp8LqdCAMGKmq\na0VkILBEVSen/QkmKyUkKB/9tpmPfttM6cL5+KFXi9AdUF61yhWwW7IEOneGW24JdETGZBvpJgUR\naQKMAIoAFUWkIfCQqj6W3ntVdSowNdm5l1O5tq03AZvzFxufwB3DFrBsx2FaVi/Op3c14qKCeQMd\nVsYMGeI2vLnoIrdN5m232SI0Y7KQN91HH+P2Zz4AoKorAVv9EyL2HztD1xF/sWzHYa5vUJavHmga\nmglBPUNJ9eq5Sqbr1sHtt1tCMCaLedN9lEtVtyfrd473UTwmC32/ZCdvT9tIzPEzPNWhJo9eVSPQ\nIZ2/EyfgxRfdQPI777gtMlv8HtTkAAAWOklEQVS3DnRUxmRb3iSFnZ4uJPWsUn4M2OTbsExmxMUn\n0HPMUmZtiKZc0QKMuv9y2l5SKtBhnb/ffoMePWDrVjfF1ArYGeNz3iSF3rgupIrAPmCm55wJQqdj\n4+nz3QpmbYima7NKvHxDHfKE+XKSmQ8cPuzqFI0YATVqwJw50KpV+u8zxmSaN4vXonHTSU2QW7/n\nKI98s4ytMSfo1aYa/a+tFeiQMmbfPle36Nln4b//hQIFAh2RMTmGN7OPPifJSuRzVLWnTyIyGfLr\n2r30GbeCk2fjefPm+tzVJMRWfp9LBE88AZdc4hallQjxDX2MCUHedB/NTHKcH+jCP2samQD7Y9N+\neo5ZCsBrN9ULrYSgCt9845LB8ePQqZPrMrKEYExAeNN9NC7pcxEZA8zwWUTGawkJyoczN/HxrAhy\nCfzatw3VS10Y6LC8t2MH9OoFv/wCzZv/PYZgjAmYjNQ+qgJUyupAzPl7bOxyfl69h1plCjGy++Vc\nXDSE+t7PFbCLjoaPP3bbY1oBO2MCzpsxhUP8PaaQCzgIpLqLmvGPUfO28vPqPdweXp5BtzQInfpF\nkZFQqZJbd/D551CtGlSuHOiojDEeac5VFPeTpiFQ0vO4SFWrqur3/gjOpGz5jkMM+GkdhfLl5s2b\nQyQhxMXBoEFQp47bHxmgXTtLCMYEmTSTgqoqMElV4z0PK1sdYOMW7+DWoQvIJTC+d3PCcoVAQlix\nwm1607+/G0i+7bZAR2SMSYU3q5oWiUgjn0di0qSqPDdxNc9OWE3xgnmZ3qc1tcoUDnRY6fv0U7j8\ncti1C374ASZOhLJlAx2VMSYVqY4piEhuVY0DrgB6iMgW4ARuRzVVVUsUfvTD0ijGLtpBs6rF+LJ7\nEwrkDfJB2XMlKRo0gHvugfffh2LFAh2VMSYdaQ00LwIaATf5KRaTip9X7eGZCasA+OqBpuTNHcRl\nK44fhxdegDx53CY4VsDOmJCSVlIQAFXd4qdYTAq+W7SD/hNXUzBvGN/0aBbcCeHXX6FnT7f+wArY\nGROS0koKJUWkX2ovqur7PojHJPHZ71sYNG0D+fPkYlqf1sG7U9qhQ9CvH4wa5UpUzJkDV1wR6KiM\nMRmQVlIIAy7E02Iw/vXL6j0MmraBQvlz8+ezV1GkQJ5Ah5S66Gg3iPzcc/Dyy5A/f6AjMsZkUFpJ\nYY+qDvRbJCbRoq0H6f3NMgrly820Pq2DMyHs3Qtjx0Lfvn8XsCtePNBRGWMyKa0OamshBMCirQe5\nb+RfAEx9ohXlgq10hSqMHu0WoT33HGze7M5bQjAmW0irpdDOb1EY9h09zUOjl7B61xEA/veflsE3\nhrBtGzz8sBtQbtkSvvjCCtgZk82kmhRU9aA/A8nJFm09yO3DFgDQpEoxPrzj0uArbhcXB1deCTEx\nrkxFr16QK4hnQhljMiQjVVJNFlqw5QB3fb4QgKH3NqJjvSBb7RsRAVWquAJ2I0dC1aquoJ0xJluy\nX/UCaM2uI3QbuQiArx5oElwJITYW3ngD6tb9u4DdlVdaQjAmm7OWQoD8FXmAO4a7FsLwro1pXbNk\ngCNKYtkyePBBV8juttvgjjsCHZExxk8sKfiZqvLmLxsYPieS3LmESY+0pH75IoEO628ff+wWopUs\n6YrXdekS6IiMMX5kScHPXv5xLWMWbqdqiYKMur8JFYsHyQyjcyUpLrsM7rsP3nsPLroo0FEZY/zM\nkoKfnDobz5u/rGfMwu00LF+ESY+0JFcw7IVw7Jhbb5Avn0sErVq5hzEmR7KBZj957ed1fLVgO9fW\nK8MPvVsER0KYNg3q1YMhQ1xLwfZQMibHs6TgYwkJSv8Jq/jmrx3c2PBiPru3MXnCAvyf/cAB6NYN\nrr0WChaEefPcfgdW0dSYHM+Sgg/FxSfw8uQ1fLd4J02qFOPd2xoGOiTnwAGYNAleegmWL4fmzQMd\nkTEmSPg0KYhIRxHZKCIRItI/hdf7icg6EVklIr+JSLaZBK+q9ByzlK8X7qBrs0qM6xngvRD27HGb\n3qhCzZqwfTsMHOjGEowxxsNnP6VEJAwYDFwL1AHuEpE6yS5bDoSragPgB+BtX8Xjb1/O28asDdF0\na16JV2+qhwSqa0bVrUSuXdu1DCIi3HmbWWSMSYEvf3VtAkSoaqSqngW+AzonvUBVZ6vqSc/ThUB5\nH8bjN39s2s/rU9fTsHwRXrgueR70o61boUMHtxCtYUNYudIK2Blj0uTLKanlgJ1JnkcBTdO4/kHg\nl5ReEJGeQE+AihUrZlV8PrF+z9HE0hWf3t0ocF1GcXFw1VVu/OCzz9w2mVbAzhiTDl8mhZT6S1Kc\n8ygi9wLhQJuUXlfV4cBwgPDw8KCdN7lp37HEaqeTHw1Q6evNm13Ruty54csvoVo1qFDB/3EYY0KS\nL391jAKS/jQqD+xOfpGItAdeAG5U1TM+jMenIvcfp9NHczl1Np5R919Og/JF/RtAbCy89ppbd/Dp\np+5c27aWEIwx58WXLYXFQA0RqQLsAu4E7k56gYhcBgwDOqpqtA9j8amEBOWh0UuIS1DG9mhG82p+\n3oVsyRI3brBqFdx5J9x1l3+/3xiTbfispaCqccCjwHRgPfC9qq4VkYEicqPnsneAC4HxIrJCRCb7\nKh5fGjR9A5ExJ+h3dU3/J4SPPoKmTd3mNz/+6PZNLlXKvzEYY7INn9Y+UtWpwNRk515Octzel9/v\nDz+v2sOwPyJpU7Mkj11V3X9ffK6AXXi4ayW8/TYU9XOXlTEm27GCeJkQfew0T/+wkhqlLmTovY39\nsxbh6FF49lnInx8++MDtldyype+/1xiTI9gcxUx4bcp6zsQl8Mndl1Egb5jvv3DqVLcT2vDhbnaR\nFbAzxmQxSwoZtHb3ESav3M1DV1ShVpnCvv2ymBi491647jooUgTmz4d33rECdsaYLGdJIQMSEpSn\nxq8il8CDV1Tx/RceOgQ//QT//a/bKrNpWmsAjTEm42xMIQOG/B7B+j1HaVzpIkoVzu+bL9m1C775\nBp5+2pWm2L7dBpKNMT5nLYXzNH3tXt79dRPFCuZl/MM+KDmtCp9/DnXqwIABsGWLO28JwRjjB5YU\nzsPS7Qd5eMxSyhUtwMx+bbJ+97QtW6BdO1enqFEjtxituh+nuRpjcjzrPvJSfILywqQ1AHz9UFOK\nFcybtV8QF+cSwsGDMGwYPPSQFbAzxvidJQUvPT1+JRv2HuPZjrWoUqJg1n3wxo2uaF3u3DB6tDsu\nny0qiBtjQpD9KuqFoX9sYeLyXdzVpCK921bLmg89exZeeQXq14fBg925Nm0sIRhjAspaCukYs3A7\nb/2ygdKF8/HidbWz5kMXLXKlKdasgbvvhnvuyZrPNcaYTLKWQhq2xZzgzanrqVqyIHOfuYqC+bIg\nh374ITRv/vfag2++gRIlMv+5xhiTBSwppCIhQen3/QpOno1nyD1ZsIPauZIUTZpAjx6wdi1cf33m\nAzXGmCxk3Uep+G7xTpbtOJz5MhZHjsAzz0CBAq6V0KKFexhjTBCylkIKtsac4L+T19CwQlGe65SJ\ncYSffnKL0L74AvLlswJ2xpigZ0khmQPHz9BlyDxU4d1bGxCWkQVq+/e7AeQbb4TixWHhQhg0yArY\nGWOCniWFJFSVnmOWcvhkLINuaUCN0oUy9kFHjrgy16+84rbKvPzyrA3UGGN8xMYUkhjx51aWbj/E\nk1fX5JbG57leYOdO+Ppr6N/flabYvt2VuTbGmBBiLQWP3YdPMWjaBsoVLUCP1lW9f2NCAgwd6ja/\nee21vwvYWUIwxoQgSwq4bqOnxq8kQWHU/ZeTP4+Xu6ht3gxXXQW9e7uppqtXWwE7Y0xIs+4j4Iel\nUczfcoCnr7nE+3GEuDi4+mo4fBhGjID777eBZGNMyMvxSWHt7iM8P2k1dcoWpkcrL7qN1q93m97k\nzg1jxrgCdhdf7PtAjTHGD3J091FcfAL9xq1ERBic3qrlM2fcdpgNGsCnn7pzrVpZQjDGZCs5uqXw\n+tT1bNx3jIGd66ZdDnvhQlfAbt066NrVPYwxJhvKsS2Frxdu58t527iiegm6NquU+oXvvefKUhw7\n5tYefPWVW5BmjDHZUI5MCit2HualH9dQvdSFfNEtHElpgDghwf3ZvDn06uXKXF97rX8DNcYYP8uR\n3UeDftlAwby5Gduj2b+nnx4+DE8+CRdcAJ98YgXsjDE5So5rKSzdfogFkQfo0aoqJQvl++eL//uf\nK2A3ejQUKmQF7IwxOU6OSgqqyptT15M/Ty7ua55kHCE6Gm6/Hbp0gdKl3c5ob7xh6w6MMTlOjkoK\nczbHsGT7Ifp3rMVFBfP+/cLRozBjBrz+uksIjRoFLkhjjAkgnyYFEekoIhtFJEJE+qfwej4RGed5\n/S8RqezLeGat30eeMOHOJhVhxw6XBFRdaYodO+D55yFPHl+GYIwxQc1nSUFEwoDBwLVAHeAuEamT\n7LIHgUOqWh34ABjkq3hUlV/W7KV1teLk/3yYK2D3xht/F7ArlMEy2cYYk434sqXQBIhQ1UhVPQt8\nB3ROdk1nYLTn+AegnaQ4PzTztuw/zoXbtvDmJ4/Df/7jppquXWsF7IwxJglfTkktB+xM8jwKaJra\nNaoaJyJHgOJATFYHs2hzNF99/zLF5Sx8+SV062YDycYYk4wvWwop/cRNPsfTm2sQkZ4iskREluzf\nvz9DwZQtUYgJfd8k17q10L27JQRjjEmBL5NCFFAhyfPywO7UrhGR3EAR4GDyD1LV4aoarqrhJUuW\nzFAwV15SiicGPIBYATtjjEmVL5PCYqCGiFQRkbzAncDkZNdMBrp5jm8FZqnaijFjjAkUn40peMYI\nHgWmA2HASFVdKyIDgSWqOhkYAYwRkQhcC+FOX8VjjDEmfT6tfaSqU4Gpyc69nOT4NHCbL2Mwxhjj\nvRy1otkYY0zaLCkYY4xJZEnBGGNMIksKxhhjEllSMMYYk0hCbVmAiOwHtmfw7SXwQQmNIGf3nDPY\nPecMmbnnSqqa7urfkEsKmSEiS1Q1PNBx+JPdc85g95wz+OOerfvIGGNMIksKxhhjEuW0pDA80AEE\ngN1zzmD3nDP4/J5z1JiCMcaYtOW0loIxxpg0ZMukICIdRWSjiESISP8UXs8nIuM8r/8lIpX9H2XW\n8uKe+4nIOhFZJSK/iUilQMSZldK75yTX3SoiKiIhP1PFm3sWkds9f9drReRbf8eY1bz4t11RRGaL\nyHLPv+9OgYgzq4jISBGJFpE1qbwuIvKx57/HKhFplKUBqGq2euDKdG8BqgJ5gZVAnWTXPAIM9Rzf\nCYwLdNx+uOcrgQs8x71zwj17risEzAEWAuGBjtsPf881gOXARZ7npQIdtx/ueTjQ23NcB9gW6Lgz\nec+tgUbAmlRe7wT8gtu5shnwV1Z+f3ZsKTQBIlQ1UlXPAt8BnZNd0xkY7Tn+AWgnEtL7c6Z7z6o6\nW1VPep4uxO2EF8q8+XsGeBV4Gzjtz+B8xJt77gEMVtVDAKoa7ecYs5o396xAYc9xEf69w2NIUdU5\npLADZRKdga/UWQgUFZGyWfX92TEplAN2Jnke5TmX4jWqGgccAYr7JTrf8Oaek3oQ95tGKEv3nkXk\nMqCCqk7xZ2A+5M3fc02gpojME5GFItLRb9H5hjf3PAC4V0SicPu3POaf0ALmfP9/Py8+3WQnQFL6\njT/5FCtvrgklXt+PiNwLhANtfBqR76V5zyKSC/gA6O6vgPzAm7/n3LgupLa41uBcEamnqod9HJuv\neHPPdwGjVPU9EWmO282xnqom+D68gPDpz6/s2FKIAiokeV6efzcnE68Rkdy4JmdazbVg5809IyLt\ngReAG1X1jJ9i85X07rkQUA/4XUS24fpeJ4f4YLO3/7Z/VNVYVd0KbMQliVDlzT0/CHwPoKoLgPy4\nGkHZlVf/v2dUdkwKi4EaIlJFRPLiBpInJ7tmMtDNc3wrMEs9IzghKt179nSlDMMlhFDvZ4Z07llV\nj6hqCVWtrKqVceMoN6rqksCEmyW8+bf9P9ykAkSkBK47KdKvUWYtb+55B9AOQERq45LCfr9G6V+T\ngfs8s5CaAUdUdU9WfXi26z5S1TgReRSYjpu5MFJV14rIQGCJqk4GRuCamBG4FsKdgYs487y853eA\nC4HxnjH1Hap6Y8CCziQv7zlb8fKepwMdRGQdEA88raoHAhd15nh5z08Cn4tIX1w3SvdQ/iVPRMbi\nuv9KeMZJ/gvkAVDVobhxk05ABHASuD9Lvz+E/9sZY4zJYtmx+8gYY0wGWVIwxhiTyJKCMcaYRJYU\njDHGJLKkYIwxJpElBRO0RCReRFYkeVRO49rKqVWV9DcRCReRjz3HbUWkRZLXeonIfX6M5dJQrxpq\n/CvbrVMw2copVb000EGcL88CuXOL5NoCx4H5nteGZvX3iUhuTw2vlFyKK2syNau/12RP1lIwIcXT\nIpgrIss8jxYpXFNXRBZ5WherRKSG5/y9Sc4PE5GwFN67TUQGea5bJCLVPecriduH4tx+FBU9528T\nkTUislJE5njOtRWRKZ6WTS+gr+c7W4nIABF5SkRqi8iiZPe1ynPcWET+EJGlIjI9pQqYIjJKRN4X\nkdnAIBFpIiLzxe0pMF9ELvGsAB4I3OH5/jtEpKC4ev2LPdemVFnW5GSBrh1uD3uk9sCtyF3heUzy\nnLsAyO85roFb1QpQGU/9eeAT4B7PcV6gAFAb+AnI4zk/BLgvhe/cBrzgOb4PmOI5/gno5jl+APif\n53g1UM5zXNTzZ9sk7xsAPJXk8xOfe+6rquf4WeBF3MrV+UBJz/k7cKt4k8c5CpgChHmeFwZye47b\nAxM8x92BT5O87w3g3nPxApuAgoH+u7ZH8Dys+8gEs5S6j/IAn4rIpbikUTOF9y0AXhCR8sBEVd0s\nIu2AxsBiT5mPAkBqNaDGJvnzA89xc+Bmz/EY3B4NAPOAUSLyPTDxfG4OV8TtduAt3A//O4BLcIX8\nZnjiDANSq2szXlXjPcdFgNGeVpHiKYuQgg7AjSLylOd5fqAisP48YzfZlCUFE2r6AvuAhrjuz39t\nnqOq34rIX8B1wHQReQhXbni0qj7nxXdoKsf/ukZVe4lIU893rfAkK2+Nw9Wimug+SjeLSH1grao2\n9+L9J5IcvwrMVtUunm6r31N5jwC3qOrG84jT5CA2pmBCTRFgj7pa+V1xv0n/g4hUBSJV9WNcRckG\nwG/ArSJSynNNMUl9n+o7kvy5wHM8n78LJ94D/On5nGqq+peqvgzE8M+SxgDHcGW8/0VVt+BaOy/h\nEgS4Utclxe0LgIjkEZG6qcSZVBFgl+e4exrfPx14TDzNEHHVc41JZEnBhJohQDcRWYjrOjqRwjV3\nAGtEZAVQC7d14Tpcn/2vngHdGUBqWxjm87Q0nsC1TAAeB+73vLer5zWAd0RktWc67BzcHsJJ/QR0\nOTfQnMJ3jQPu5e/9AM7iyrkPEpGVuHGHfw2mp+Bt4E0Rmcc/E+VsoM65gWZciyIPsMoT86tefLbJ\nQaxKqjFJiNuQJ1xVYwIdizGBYC0FY4wxiaylYIwxJpG1FIwxxiSypGCMMSaRJQVjjDGJLCkYY4xJ\nZEnBGGNMIksKxhhjEv0fJDZD/WqgfJ0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1ad81b03518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "GBC_model = GradientBoostingClassifier(n_estimators=500,\n",
    "                                      learning_rate=0.01)\n",
    "GBC_model.fit(X_train,y_train)\n",
    "pred_GBC_test_proba = GBC_model.predict_proba(X_test)[:,1]\n",
    "\n",
    "fpr, tpr, _ = roc_curve(y_test, pred_GBC_test_proba,pos_label=1)\n",
    "#df = pd.DataFrame(dict(fpr=fpr, tpr=tpr))\n",
    "plt.plot(fpr,tpr)\n",
    "plt.plot([0,1],[0,1],'r--')\n",
    "plt.ylabel('True positive rate')\n",
    "plt.xlabel('False positive rate')\n",
    "print(\"roc_auc score: {0}\".format(roc_auc_score(y_test,pred_GBC_test_proba)))\n",
    "\n",
    "print(\"Predicting on validation data ...\")\n",
    "pred_GBC_valid = GBC_model.predict(data_valid[features])\n",
    "print(\"This model predicts {0:.2f}% of positive results on the validation data\".format(100*pred_GBC_valid.sum()/len(data_valid)))\n",
    "pred_GBC_proba_valid = GBC_model.predict_proba(data_valid[features])[:,1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here are the features importance according to this model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature: ps_ind_01\t- Importance: 0.026019666579099678\n",
      "\n",
      "Feature: ps_ind_03\t- Importance: 0.14066167185716505\n",
      "\n",
      "Feature: ps_ind_05_cat\t- Importance: 0.053896923400882535\n",
      "\n",
      "Feature: ps_ind_06_bin\t- Importance: 0.027420938160716477\n",
      "\n",
      "Feature: ps_ind_07_bin\t- Importance: 0.02426823862943589\n",
      "\n",
      "Feature: ps_ind_08_bin\t- Importance: 0.00534659641569545\n",
      "\n",
      "Feature: ps_ind_09_bin\t- Importance: 0.007873744173453618\n",
      "\n",
      "Feature: ps_ind_11_bin\t- Importance: 0.00041410833204131724\n",
      "\n",
      "Feature: ps_ind_12_bin\t- Importance: 0.0041364677329571095\n",
      "\n",
      "Feature: ps_ind_14\t- Importance: 0.0009272295870138917\n",
      "\n",
      "Feature: ps_ind_15\t- Importance: 0.05275880805055454\n",
      "\n",
      "Feature: ps_ind_16_bin\t- Importance: 0.017060234216606626\n",
      "\n",
      "Feature: ps_ind_17_bin\t- Importance: 0.06566746591199105\n",
      "\n",
      "Feature: ps_reg_01\t- Importance: 0.04381313127117623\n",
      "\n",
      "Feature: ps_reg_02\t- Importance: 0.053873477047355574\n",
      "\n",
      "Feature: ps_car_06_cat\t- Importance: 0.0013624255057091798\n",
      "\n",
      "Feature: ps_car_11\t- Importance: 0.011250299142833918\n",
      "\n",
      "Feature: ps_car_12\t- Importance: 0.005296970270348556\n",
      "\n",
      "Feature: ps_car_13\t- Importance: 0.16290526627931715\n",
      "\n",
      "Feature: ps_car_14\t- Importance: 0.04127289236990411\n",
      "\n",
      "Feature: ps_car_15\t- Importance: 0.0019391408500353659\n",
      "\n",
      "Feature: ps_calc_02\t- Importance: 0.007653516396438271\n",
      "\n",
      "Feature: ps_calc_05\t- Importance: 0.002558154359837961\n",
      "\n",
      "Feature: ps_calc_06\t- Importance: 0.00019371823732311852\n",
      "\n",
      "Feature: ps_calc_07\t- Importance: 0.007554035963159941\n",
      "\n",
      "Feature: ps_calc_08\t- Importance: 0.0016064843990663412\n",
      "\n",
      "Feature: ps_calc_10\t- Importance: 0.0004991840190387553\n",
      "\n",
      "Feature: ps_calc_11\t- Importance: 0.000994970167622205\n",
      "\n",
      "Feature: ps_calc_12\t- Importance: 0.00018699478491157688\n",
      "\n",
      "Feature: ps_calc_13\t- Importance: 0.0002123357581462533\n",
      "\n",
      "Feature: ps_calc_14\t- Importance: 0.0009294500890790268\n",
      "\n",
      "Feature: ps_calc_18_bin\t- Importance: 0.0009906443747860721\n",
      "\n",
      "Feature: ps_ind_02_cat_0\t- Importance: 0.003260971038575709\n",
      "\n",
      "Feature: ps_ind_02_cat_1\t- Importance: 0.0028190750814433194\n",
      "\n",
      "Feature: ps_ind_04_cat_0\t- Importance: 0.00028066328398255764\n",
      "\n",
      "Feature: ps_ind_05_cat_0\t- Importance: 0.022380514554597403\n",
      "\n",
      "Feature: ps_ind_05_cat_2\t- Importance: 0.017590079726470483\n",
      "\n",
      "Feature: ps_ind_05_cat_6\t- Importance: 0.007010107872695604\n",
      "\n",
      "Feature: ps_car_01_cat_6\t- Importance: 0.003988732330565895\n",
      "\n",
      "Feature: ps_car_01_cat_7\t- Importance: 0.03249594149906273\n",
      "\n",
      "Feature: ps_car_01_cat_9\t- Importance: 0.01513213275893873\n",
      "\n",
      "Feature: ps_car_01_cat_10\t- Importance: 0.0002373953332082852\n",
      "\n",
      "Feature: ps_car_01_cat_11\t- Importance: 0.0011224227004250102\n",
      "\n",
      "Feature: ps_car_04_cat_0\t- Importance: 0.01215580623121862\n",
      "\n",
      "Feature: ps_car_04_cat_2\t- Importance: 0.012066159873860014\n",
      "\n",
      "Feature: ps_car_04_cat_5\t- Importance: 0.00049082963068739\n",
      "\n",
      "Feature: ps_car_04_cat_7\t- Importance: 0.0003727192838096859\n",
      "\n",
      "Feature: ps_car_06_cat_4\t- Importance: 0.00033899245170758934\n",
      "\n",
      "Feature: ps_car_06_cat_5\t- Importance: 0.0006239152490023838\n",
      "\n",
      "Feature: ps_car_06_cat_9\t- Importance: 0.007782466291996507\n",
      "\n",
      "Feature: ps_car_06_cat_17\t- Importance: 0.0016711362700610485\n",
      "\n",
      "Feature: ps_car_07_cat_0\t- Importance: 0.025640218489688022\n",
      "\n",
      "Feature: ps_car_09_cat_0\t- Importance: 0.008607091808417193\n",
      "\n",
      "Feature: ps_car_09_cat_1\t- Importance: 0.03460095788198605\n",
      "\n",
      "Feature: ps_car_11_cat_0\t- Importance: 0.000362962207378494\n",
      "\n",
      "Feature: ps_car_11_cat_20\t- Importance: 0.0025034193932685165\n",
      "\n",
      "Feature: ps_car_11_cat_30\t- Importance: 0.00020611790169240915\n",
      "\n",
      "Feature: ps_car_11_cat_33\t- Importance: 0.0007961422940832321\n",
      "\n",
      "Feature: ps_car_11_cat_40\t- Importance: 0.0014179782691112158\n",
      "\n",
      "Feature: ps_car_11_cat_45\t- Importance: 0.0012647391757083456\n",
      "\n",
      "Feature: ps_car_11_cat_51\t- Importance: 0.00010133429114860712\n",
      "\n",
      "Feature: ps_car_11_cat_60\t- Importance: 0.0036117211569125806\n",
      "\n",
      "Feature: ps_car_11_cat_64\t- Importance: 0.000768994374771149\n",
      "\n",
      "Feature: ps_car_11_cat_78\t- Importance: 0.0006076156954583794\n",
      "\n",
      "Feature: ps_car_11_cat_79\t- Importance: 0.0032165310738494664\n",
      "\n",
      "Feature: ps_car_11_cat_82\t- Importance: 0.0006502068972239768\n",
      "\n",
      "Feature: ps_car_11_cat_89\t- Importance: 0.0012207297904447887\n",
      "\n",
      "Feature: ps_car_11_cat_96\t- Importance: 0.0005053528971774307\n",
      "\n",
      "Feature: ps_car_11_cat_101\t- Importance: 0.0005526406056684549\n",
      "\n"
     ]
    }
   ],
   "source": [
    "non_zero_idx = np.array(GBC_model.feature_importances_.nonzero())\n",
    "\n",
    "for n in non_zero_idx[0]:\n",
    "    print(\"Feature: {0}\\t- Importance: {1}\\n\".format(features[n], GBC_model.feature_importances_[n]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Data for submission\n",
    "data_valid = pd.read_csv(\"test.csv\")\n",
    "pred = logit_model.predict(data_valid[features])\n",
    "\n",
    "df = pd.DataFrame()\n",
    "df[\"id\"] = data_valid[\"id\"]\n",
    "df[\"target\"] = pred\n",
    "\n",
    "df.to_csv(\"pred1.csv\", index=None, sep=\",\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}