dask-geopandas-basics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import geopandas as gpd\n",
    "import dask_geopandas as dg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import shapely\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "n = 500000\n",
    "\n",
    "points = [shapely.geometry.Point(random.random(), random.random()) for i in range(n)]\n",
    "points_df = gpd.GeoDataFrame({'value': np.random.random(n),\n",
    "                              'geometry': points})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f4a762fdac8>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQIAAAD8CAYAAACcoKqNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADoNJREFUeJzt3X+I5PV9x/Hny7PWtv7I4SU0eLeeoSfkIgVlsLGWqsSW\n08JdKZqeoInFesSqLTUULAaTmr9qaUOj16aLlZhC/JFQzBIutSQ9MbWeuTk0xju5srkkdTHURI0F\nxajdd/+YMR3HvZuv7ndmZ/H5gA/Md+Z933nf7M1rv9/vzr03VYWkd7ajVroBSSvPIJBkEEgyCCRh\nEEjCIJCEQSAJg0ASBoEk4OiVeuJ169bVxo0bV+rppXeEffv2/biq3j2qbsWCYOPGjXS73ZV6eukd\nIckPmtR5aiDJIJBkEEjCIJCEQSCJBkGQ5I4kzyR54jCPJ8lnk8wneTzJme23KWmcmhwRfB7YcoTH\nLwQ29dcO4O+X35akSRoZBFX1IPDcEUq2AV+onj3Au5K8t60GJY1fGx8oOhl4amB7oX/fD4cLk+yg\nd9TAzMxMo50f9wvH8eLLLy6/S02fAG2NzGxzX0AC0zzOc+3atTz33JG+P781bQRBlrhvyZewqmaB\nWYBOp9PoZX7x5Re585P3vP3uhnz0L36fL+z8eiv7+sg1F3D73f/eyr4A/nD7b7DzX/6jtf1ds+XX\n+czDD7W2vz89+xxuPbC7tf1dt/l8/vmp+1rZ1+9t+F32P9veWekHTrqa//2fv21tf2tO+JPW/x23\nqY2fGiwAGwa21wNPt7BfSRPSRhDMAR/p//Tgg8ALVfWm0wJJ02vkqUGSu4DzgHVJFoBPAj8HUFWf\nA3YBFwHzwEvAH4yrWUnjMTIIqurSEY8XcE1rHUmaOD9ZKMkgkGQQSMIgkIRBIAmDQBIGgSQMAkkY\nBJIwCCRhEEjCIJCEQSAJg0ASkFqhwWydTqea/BLUZKlJaNI7WxIWFxeb1O2rqs6ouhX7bchvxbTO\njmtz/iE4A3E52px/CKtjBmKbPDWQZBBIMggkYRBIwiCQhEEgCYNAEgaBJAwCSRgEkjAIJGEQSMIg\nkIRBIImGQZBkS5KDSeaT3LDE4zNJdid5NMnjSS5qv1VJ4zIyCJKsAXYCFwKbgUuTbB4q+wRwb1Wd\nAWwH/q7tRiWNT5MjgrOA+ao6VFWvAHcD24ZqCjihf/tE4On2WpQ0bk0mFJ0MPDWwvQD82lDNp4B/\nTXId8EvABa10J2kyquqIC7gEuH1g+3Lg1qGa64GP92+fDRwAjlpiXzuALtCdmZmpJugdbbhcroGV\npOn7pzvqPV5VjY4IFoANA9vrefOh/5XAFoCqejjJscA64JnBoqqaBWahN7y0wXMDTO3suDbnH4Iz\nEJejzfmHsDpmILapyTWCvcCmJKcmOYbexcC5oZr/Aj4EkOT9wLHAj9psVNL4jAyCqnoNuBa4H3iS\n3k8H9ie5OcnWftnHgauSfBu4C7iiVmpOuqS3rNE486raBewauu+mgdsHgHPabU3SpPjJQkkGgSSD\nQBIGgSQMAkkYBJIwCCRhEEjCIJCEQSAJg0ASBoEkDAJJGASSMAgkAVmp+SGdTqe63e7IuiQT6EZa\nXZKwuLjYpG5fVXVG1TUaTLLSpnV2XJvzD8EZiMvR5vxDWB0zENvkqYEkg0CSQSAJg0ASBoEkDAJJ\nGASSMAgkYRBIwiCQhEEgCYNAEgaBJAwCSTQMgiRbkhxMMp/khsPUfDjJgST7k3yx3TYljdPIeQRJ\n1gA7gd8CFoC9Seaq6sBAzSbgz4Fzqur5JO8ZV8OS2tfkiOAsYL6qDlXVK8DdwLahmquAnVX1PEBV\nPdNum5LGqqqOuICLgdsHti8HbhuquQ+4BXgI2ANsOcy+dgBdoDszM1NNAC6Xa2glafr+6Y56j1dV\no1FlSw0NrKHto4FNwHnAeuCbSU6vqp+84Q9VzQKz0JtZ2OC5AaZ2ZFSbY8/A0WfL0ebYM1gdo8/a\n1OTUYAHYMLC9Hnh6iZqvVNWrVfU94CC9YJC0CjQJgr3ApiSnJjkG2A7MDdXcB5wPkGQdcBpwqM1G\nJY3PyCCoqteAa4H7gSeBe6tqf5Kbk2ztl90PPJvkALAb+LOqenZcTUtqV6Nx5lW1C9g1dN9NA7cL\nuL6/JK0yfrJQkkEgySCQhEEgCYNAEgaBJAwCSRgEkjAIJGEQSMIgkIRBIAmDQBIGgSQg/VmCE9fp\ndKrb7Y6sS5aalCa9syVhcXGxSd2+quqMqms0j2ClTevsuDbnH4IzEJejzfmHsDpmILbJUwNJBoEk\ng0ASBoEkDAJJGASSMAgkYRBIwiCQhEEgCYNAEgaBJAwCSRgEkmgYBEm2JDmYZD7JDUeouzhJJRn5\n/58lTY+RQZBkDbATuBDYDFyaZPMSdccDfww80naTksaryRHBWcB8VR2qqleAu4FtS9R9GrgFeLnF\n/iRNQJMgOBl4amB7oX/fzyQ5A9hQVV9tsTdJk1JVR1zAJcDtA9uXA7cObB8FPABs7G8/AHQOs68d\nQBfozszMVBOAy+UaWkmavn+6o97jVdVoZuECsGFgez3w9MD28cDpwAP9QaO/DMwl2VpVb5hOWlWz\nwCz0hpc2eG6AqZ0d1+b8Q3AG4nK0Of8QVscMxDY1OTXYC2xKcmqSY4DtwNzrD1bVC1W1rqo2VtVG\nYA/wphCQNL1GBkFVvQZcC9wPPAncW1X7k9ycZOu4G5Q0fo3GmVfVLmDX0H03Hab2vOW3JWmS/GSh\nJINAkkEgCYNAEgaBJAwCSRgEkjAIJGEQSMIgkIRBIAmDQBIGgSQMAkkYBJKA9GcJTlyn06lud/QQ\no/74M0kDkrC4uNikbl9Vjfw9I40Gk6y0aZ0d1+b8Q3AG4nK0Of8QVscMxDZ5aiDJIJBkEEjCIJCE\nQSAJg0ASBoEkDAJJGASSMAgkYRBIwiCQhEEgCYNAEg2DIMmWJAeTzCe5YYnHr09yIMnjSb6R5JT2\nW5U0LiODIMkaYCdwIbAZuDTJ5qGyR4FOVf0q8GXglrYblTQ+TY4IzgLmq+pQVb0C3A1sGyyoqt1V\n9VJ/cw+wvt02JY1VVR1xARcDtw9sXw7cdoT624BPHOaxHUAX6M7MzFQTgMvlGlpJmr5/uqPe41XV\naFTZUkMDa8nC5DKgA5y71ONVNQvMQm9mYYPnBpjakVFtjj0DR58tR5tjz2B1jD5rU5MgWAA2DGyv\nB54eLkpyAXAjcG5V/bSd9iRNQpNrBHuBTUlOTXIMsB2YGyxIcgbwD8DWqnqm/TYljdPIIKiq14Br\ngfuBJ4F7q2p/kpuTbO2X/RVwHPClJI8lmTvM7iRNoUbjzKtqF7Br6L6bBm63e8IiaaL8ZKEkg0CS\nQSAJg0ASBoEkDAJJGASSMAgkYRBIwiCQhEEgCYNAEgaBJAwCSUD6swQnrtPpVLfbHVmXLDUpTXpn\nS8Li4mKTun1V1RlV12gewUqb1tlxbc4/BGcgLkeb8w9hdcxAbJOnBpIMAkkGgSQMAkkYBJIwCCRh\nEEjCIJCEQSAJg0ASBoEkDAJJGASSMAgk0TAIkmxJcjDJfJIblnj855Pc03/8kSQb225U0viMDIIk\na4CdwIXAZuDSJJuHyq4Enq+qXwE+A/xl241KGp8mRwRnAfNVdaiqXgHuBrYN1WwD7uzf/jLwoTha\nSFo1mgTBycBTA9sL/fuWrKmq14AXgJPaaFDSBFTVERdwCXD7wPblwK1DNfuB9QPb3wVOWmJfO4Au\n0J2ZmakmAJfLNbTWrl3b9P3THfUer6pGMwsXgA0D2+uBpw9Ts5DkaOBE4LnhHVXVLDALveGlDZ77\n9QCRNEZNTg32ApuSnJrkGGA7MDdUMwd8tH/7YuDfynewtGqMPCKoqteSXAvcD6wB7qiq/UlupnfY\nMQf8I/BPSebpHQlsH2fTktrVaJx5Ve0Cdg3dd9PA7ZfpXUuQtAr5yUJJBoEkg0ASBoEkDAJJrOBv\nQ07yI+AHDUrXAT8ecztv1zT3Bva3HNPcGzTv75SqeveoohULgqaSdJv8WueVMM29gf0txzT3Bu33\n56mBJINA0uoIgtmVbuAIprk3sL/lmObeoOX+pv4agaTxWw1HBJLGbGqCYJoHpDbo7fokB5I8nuQb\nSU6ZVG9N+huouzhJJZnY1fAmvSX5cP/125/ki5PqrUl/SWaS7E7yaP/re9EEe7sjyTNJnjjM40ny\n2X7vjyc5820/WZPpJeNe9P5783eB9wHHAN8GNg/V/BHwuf7t7cA9U9Tb+cAv9m9fPanemvbXrzse\neBDYA3SmpTdgE/AosLa//Z5peu3onYtf3b+9Gfj+BPv7TeBM4InDPH4R8DUgwAeBR97uc03LEcE0\nD0gd2VtV7a6ql/qbe+hNcZqUJq8dwKeBW4CXp6y3q4CdVfU8QFU9M2X9FXBC//aJvHk619hU1YMs\nMelrwDbgC9WzB3hXkve+neealiCY5gGpTXobdCW9lJ6Ukf0lOQPYUFVfnWBf0Oy1Ow04LclDSfYk\n2TKx7pr19yngsiQL9GZyXDeZ1hp5q/82D6vRYJIJWOo7+/CPM5rUjEPj501yGdABzh1rR0NPu8R9\nP+svyVH0ftfEFZNqaECT1+5oeqcH59E7kvpmktOr6idj7g2a9Xcp8Pmq+uskZ9ObxHV6VS2Ov72R\nWntPTMsRwVsZkMqRBqSuUG8kuQC4EdhaVT+dQF+vG9Xf8cDpwANJvk/vXHJuQhcMm35dv1JVr1bV\n94CD9IJhEpr0dyVwL0BVPQwcS+9z/tOg0b/NRiZ14WPERZGjgUPAqfz/RZsPDNVcwxsvFt47Rb2d\nQe+i06ZpfO2G6h9gchcLm7x2W4A7+7fX0TvUfdMo/BXs72vAFf3b7++/0TLBr+9GDn+x8Hd448XC\nb73t55nUX6jBX/gi4D/7b6gb+/fdTO87LPSS+EvAPPAt4H1T1NvXgf8GHuuvuWl67YZqJxYEDV+7\nAH8DHAC+A2yfpteO3k8KHuqHxGPAb0+wt7uAHwKv0vvufyXwMeBjA6/dzn7v31nO19VPFkqammsE\nklaQQSDJIJBkEEjCIJCEQSAJg0ASBoEk4P8AnsO5/8TMv2UAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f4a762fd0b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "delta = 0.1\n",
    "inds = np.arange(0, 1, delta)\n",
    "grid = [shapely.geometry.Polygon([(x, y), (x + delta, y),\n",
    "                 (x + delta, y + delta), (x, y + delta)])\n",
    "        for x in inds\n",
    "        for y in inds]\n",
    "grid_df = gpd.GeoDataFrame({'x': np.concatenate([inds] * 10),\n",
    "                            'y': np.repeat(inds, 10),\n",
    "                            'geometry': grid})\n",
    "grid_df.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.78 s, sys: 24 ms, total: 4.81 s\n",
      "Wall time: 4.81 s\n"
     ]
    }
   ],
   "source": [
    "%time df = dg.partition(points_df, grid_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<GeoDataFrame: from-pandas, npartitions=100>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I am densified (5 elements)\n",
      "I am densified (5 elements)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>value</th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>478100</th>\n",
       "      <td>0.843377</td>\n",
       "      <td>POINT (0.009886932833357287 0.0999559209560178)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>348292</th>\n",
       "      <td>0.980777</td>\n",
       "      <td>POINT (0.08504669806988341 0.09995216120167449)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159210</th>\n",
       "      <td>0.439273</td>\n",
       "      <td>POINT (0.02733365841219149 0.09993243894068693)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352690</th>\n",
       "      <td>0.242622</td>\n",
       "      <td>POINT (0.07595043840557492 0.09993124960349686)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462197</th>\n",
       "      <td>0.142140</td>\n",
       "      <td>POINT (0.03244849193210575 0.09989968182099018)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           value                                         geometry\n",
       "478100  0.843377  POINT (0.009886932833357287 0.0999559209560178)\n",
       "348292  0.980777  POINT (0.08504669806988341 0.09995216120167449)\n",
       "159210  0.439273  POINT (0.02733365841219149 0.09993243894068693)\n",
       "352690  0.242622  POINT (0.07595043840557492 0.09993124960349686)\n",
       "462197  0.142140  POINT (0.03244849193210575 0.09989968182099018)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Time geometry operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"100.0\" height=\"100.0\" viewBox=\"-0.14800000000000002 -0.14800000000000002 1.2960000000000003 1.2960000000000003\" preserveAspectRatio=\"xMinYMin meet\"><g transform=\"matrix(1,0,0,-1,0,1.0)\"><path fill-rule=\"evenodd\" fill=\"#66cc99\" stroke=\"#555555\" stroke-width=\"0.025920000000000006\" opacity=\"0.6\" d=\"M 0.9292893218813453,1.0707106781186548 L 0.9365606715836355,1.0773010453362737 L 0.9444429766980399,1.0831469612302547 L 0.9528603263174004,1.0881921264348355 L 0.9617316567634911,1.0923879532511287 L 0.9709715322745539,1.095694033573221 L 0.9804909677983873,1.0980785280403231 L 0.9901982859670441,1.0995184726672198 L 1.0,1.1 L 1.009801714032956,1.0995184726672198 L 1.019509032201613,1.0980785280403231 L 1.0290284677254462,1.0956940335732208 L 1.038268343236509,1.0923879532511287 L 1.0471396736825997,1.0881921264348355 L 1.0555570233019602,1.0831469612302544 L 1.0634393284163646,1.0773010453362737 L 1.0707106781186548,1.0707106781186548 L 1.0773010453362737,1.0634393284163646 L 1.0831469612302544,1.0555570233019602 L 1.0881921264348355,1.0471396736825997 L 1.0923879532511287,1.038268343236509 L 1.0956940335732208,1.0290284677254462 L 1.0980785280403231,1.019509032201613 L 1.0995184726672196,1.0098017140329563 L 1.1,1.0000000000000002 L 1.0995184726672198,0.9901982859670442 L 1.0980785280403231,0.9804909677983874 L 1.095694033573221,0.970971532274554 L 1.0923879532511287,0.9617316567634913 L 1.0881921264348358,0.9528603263174006 L 1.0831469612302547,0.9444429766980401 L 1.077301045336274,0.9365606715836358 L 1.070710678118655,0.9292893218813456 L 0.07071067811865475,-0.07071067811865475 L 0.06343932841636445,-0.0773010453362738 L 0.05555702330196012,-0.0831469612302546 L 0.047139673682599664,-0.08819212643483557 L 0.03826834323650888,-0.09238795325112872 L 0.029028467725446146,-0.09569403357322093 L 0.019509032201612746,-0.09807852804032308 L 0.009801714032955988,-0.09951847266721969 L -6.049014748177263e-17,-0.1 L -0.00980171403295611,-0.09951847266721969 L -0.019509032201612864,-0.09807852804032305 L -0.02902846772544626,-0.09569403357322089 L -0.03826834323650899,-0.09238795325112868 L -0.047139673682599775,-0.08819212643483551 L -0.0555570233019602,-0.08314696123025456 L -0.06343932841636454,-0.07730104533627372 L -0.07071067811865475,-0.07071067811865477 L -0.07730104533627367,-0.06343932841636458 L -0.08314696123025449,-0.05555702330196029 L -0.08819212643483548,-0.04713967368259983 L -0.09238795325112865,-0.03826834323650903 L -0.09569403357322087,-0.029028467725446323 L -0.09807852804032302,-0.019509032201612948 L -0.09951847266721968,-0.009801714032956217 L -0.1,-1.8988215193149856e-16 L -0.09951847266721972,0.009801714032955839 L -0.0980785280403231,0.019509032201612576 L -0.09569403357322098,0.029028467725445962 L -0.09238795325112881,0.038268343236508684 L -0.08819212643483568,0.04713967368259945 L -0.08314696123025475,0.0555570233019599 L -0.07730104533627397,0.06343932841636422 L -0.07071067811865508,0.07071067811865443 L 0.9292893218813453,1.0707106781186548 z\" /></g></svg>"
      ],
      "text/plain": [
       "<shapely.geometry.polygon.Polygon at 0x7f4a741f6b38>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "line = shapely.geometry.LineString([(0, 0), (1, 1)])\n",
    "line.buffer(0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.98 s, sys: 0 ns, total: 1.98 s\n",
      "Wall time: 1.98 s\n"
     ]
    }
   ],
   "source": [
    "%time _ = points_df.distance(line.buffer(0.1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.35 s, sys: 4 ms, total: 3.36 s\n",
      "Wall time: 874 ms\n"
     ]
    }
   ],
   "source": [
    "%time _ = df.distance(line.buffer(0.1)).compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Time buffering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 16.2 s, sys: 236 ms, total: 16.4 s\n",
      "Wall time: 16.4 s\n"
     ]
    }
   ],
   "source": [
    "%time _ = points_df.buffer(0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 24 s, sys: 564 ms, total: 24.5 s\n",
      "Wall time: 6.92 s\n"
     ]
    }
   ],
   "source": [
    "%time _ = df.buffer(0.1).compute()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}