201703_PandasToDictToVectorizer

201703_PandasToDictToVectorizer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>id</th>\n",
       "      <th>level1_location</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>IID1</td>\n",
       "      <td>01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11</td>\n",
       "      <td>IID2</td>\n",
       "      <td>02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12</td>\n",
       "      <td>IID4</td>\n",
       "      <td>03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  category    id level1_location\n",
       "0       10  IID1              01\n",
       "1       11  IID2              02\n",
       "2       12  IID4              03"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_features_df = pd.DataFrame(\n",
    "    [{\"id\": \"IID1\", \"category\": \"10\", \"level1_location\": \"01\"},\n",
    "     {\"id\": \"IID2\", \"category\": \"11\", \"level1_location\": \"02\"},\n",
    "     {\"id\": \"IID4\", \"category\": \"12\", \"level1_location\": \"03\"}])\n",
    "\n",
    "id_encoder = LabelEncoder()\n",
    "id_encoder.fit([\"IID1\", \"IID2\", \"IID3\", \"IID4\"])\n",
    "\n",
    "item_features_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>level1_location</th>\n",
       "      <th>feature_dict</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>01</td>\n",
       "      <td>{'category': '10', 'level1_location': '01'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11</td>\n",
       "      <td>02</td>\n",
       "      <td>{'category': '11', 'level1_location': '02'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12</td>\n",
       "      <td>03</td>\n",
       "      <td>{'category': '12', 'level1_location': '03'}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   category level1_location                                 feature_dict\n",
       "id                                                                      \n",
       "0        10              01  {'category': '10', 'level1_location': '01'}\n",
       "1        11              02  {'category': '11', 'level1_location': '02'}\n",
       "3        12              03  {'category': '12', 'level1_location': '03'}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_ids_to_features = item_features_df\\\n",
    "    .assign(id=lambda df: id_encoder.transform(df['id']))\\\n",
    "    .set_index('id')\\\n",
    "    .assign(feature_dict=lambda df: df.to_dict('records'))\n",
    "\n",
    "encoded_ids_to_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  0.,  0.,  1.,  0.,  0.],\n",
       "       [ 0.,  1.,  0.,  0.,  1.,  0.],\n",
       "       [ 0.,  0.,  1.,  0.,  0.,  1.]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v = DictVectorizer()\n",
    "\n",
    "sparse_features = v.fit_transform(encoded_ids_to_features['feature_dict'])\n",
    "\n",
    "# problem, we squeezed the row for missing IID3\n",
    "# as DictVectorizer uses the list of dict index as row index so we sq\n",
    "sparse_features.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([{'category': '10', 'level1_location': '01'},\n",
       "       {'category': '11', 'level1_location': '02'}, {},\n",
       "       {'category': '12', 'level1_location': '03'}], dtype=object)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Solution 1: building a dense numpy array of copies of the same empty {} (easy on memory)\n",
    "n_items = id_encoder.classes_.shape[0]\n",
    "dense_feature_dicts = np.repeat({}, n_items)\n",
    "\n",
    "# replace the non empty dict within this array\n",
    "dense_feature_dicts[encoded_ids_to_features.index.values] = encoded_ids_to_features['feature_dict']\n",
    "\n",
    "dense_feature_dicts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  0.,  0.,  1.,  0.,  0.],\n",
       "       [ 0.,  1.,  0.,  0.,  1.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  1.,  0.,  0.,  1.]])"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparse_features = v.fit_transform(dense_feature_dicts)\n",
    "\n",
    "sparse_features.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 1 1 2 2]\n",
      "Int64Index([0, 0, 1, 1, 3, 3], dtype='int64', name='id')\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  0.,  0.,  1.,  0.,  0.],\n",
       "       [ 0.,  1.,  0.,  0.,  1.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  1.,  0.,  0.,  1.]])"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Solution 2: correct the row indexes of the 1st sparse features matrix\n",
    "import scipy.sparse as sp\n",
    "\n",
    "zero_indexed_sparse_features = v.fit_transform(encoded_ids_to_features['feature_dict']).tocoo()\n",
    "\n",
    "print(zero_indexed_sparse_features.row)\n",
    "print(encoded_ids_to_features.index[zero_indexed_sparse_features.row])\n",
    "\n",
    "correct_row_indexes = encoded_ids_to_features.index[zero_indexed_sparse_features.row].values\n",
    "sparse_features = sp.coo_matrix(\n",
    "    (zero_indexed_sparse_features.data, (correct_row_indexes, zero_indexed_sparse_features.col)))\n",
    "\n",
    "sparse_features.toarray()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}