Skip to content

Instantly share code, notes, and snippets.

@sidhusmart

sidhusmart/Spark Dense to Pandas Sparse.ipynb

Last active December 17, 2017 12:17
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save sidhusmart/de5571cd82e7ff8397fac8360a48fb02 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save sidhusmart/de5571cd82e7ff8397fac8360a48fb02 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
Spark Dense to Pandas Sparse.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Convert the Spark encoded Dataframe to Pandas"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"oneHotPandasDF = encodedDF.toPandas()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Convert the dense representation into a sparse representation"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"def convertToSparseRepresentation(inputDF, col): \n",
"\n",
" col_sep = '_'\n",
" numCat = len(inputDF[col+\"_onehot\"][0])\n",
" recodedCols = []\n",
" \n",
" for i in range(0,numCat):\n",
" recodedCols.append(col+\"_onehot\"+col_sep+str(i))\n",
" \n",
" tempDF = pd.DataFrame(np.asarray(list(inputDF[col+\"_onehot\"].values)), columns=recodedCols)\n",
" tempDF1 = inputDF.drop(col+\"_onehot\", axis=1)\n",
" \n",
" finalDF = pd.concat([tempDF1,tempDF], axis=1)\n",
" \n",
" return finalDF"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"for col in catColumns:\n",
" oneHotPandasDF = convertToSparseRepresentation(oneHotPandasDF, col)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style>\n",
" .dataframe thead tr:only-child th {\n",
" text-align: right;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>GenderSelect</th>\n",
" <th>GenderSelect_onehot_0</th>\n",
" <th>GenderSelect_onehot_1</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Non-binary, genderqueer, or gender non-conforming</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Female</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Male</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Male</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Male</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" GenderSelect GenderSelect_onehot_0 \\\n",
"0 Non-binary, genderqueer, or gender non-conforming 0.0 \n",
"1 Female 0.0 \n",
"2 Male 1.0 \n",
"3 Male 1.0 \n",
"4 Male 1.0 \n",
"\n",
" GenderSelect_onehot_1 \n",
"0 0.0 \n",
"1 1.0 \n",
"2 0.0 \n",
"3 0.0 \n",
"4 0.0 "
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"oneHotPandasDF[['GenderSelect','GenderSelect_onehot_0','GenderSelect_onehot_1']].head()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style>\n",
" .dataframe thead tr:only-child th {\n",
" text-align: right;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>EmploymentStatus</th>\n",
" <th>EmploymentStatus_onehot_0</th>\n",
" <th>EmploymentStatus_onehot_1</th>\n",
" <th>EmploymentStatus_onehot_2</th>\n",
" <th>EmploymentStatus_onehot_3</th>\n",
" <th>EmploymentStatus_onehot_4</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Employed full-time</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Not employed, but looking for work</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Not employed, but looking for work</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Independent contractor, freelancer, or self-em...</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Employed full-time</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" EmploymentStatus \\\n",
"0 Employed full-time \n",
"1 Not employed, but looking for work \n",
"2 Not employed, but looking for work \n",
"3 Independent contractor, freelancer, or self-em... \n",
"4 Employed full-time \n",
"\n",
" EmploymentStatus_onehot_0 EmploymentStatus_onehot_1 \\\n",
"0 1.0 0.0 \n",
"1 0.0 1.0 \n",
"2 0.0 1.0 \n",
"3 0.0 0.0 \n",
"4 1.0 0.0 \n",
"\n",
" EmploymentStatus_onehot_2 EmploymentStatus_onehot_3 \\\n",
"0 0.0 0.0 \n",
"1 0.0 0.0 \n",
"2 0.0 0.0 \n",
"3 0.0 1.0 \n",
"4 0.0 0.0 \n",
"\n",
" EmploymentStatus_onehot_4 \n",
"0 0.0 \n",
"1 0.0 \n",
"2 0.0 \n",
"3 0.0 \n",
"4 0.0 "
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"oneHotPandasDF[['EmploymentStatus',\n",
" 'EmploymentStatus_onehot_0',\n",
" 'EmploymentStatus_onehot_1',\n",
" 'EmploymentStatus_onehot_2',\n",
" 'EmploymentStatus_onehot_3',\n",
" 'EmploymentStatus_onehot_4']].head()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.13"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment