l8-study.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/amitkapadia/anaconda/envs/pl/lib/python2.7/site-packages/matplotlib/__init__.py:872: UserWarning: axes.color_cycle is deprecated and replaced with axes.prop_cycle; please use the latter.\n",
      "  warnings.warn(self.msg_depr % (key, alt_key))\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "%matplotlib inline\n",
    "sns.set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df1 = pd.read_csv('l8-on-s3.txt.gz', compression='gzip', names=['date', 'time', 'bytes', 'path'],  delim_whitespace=True, usecols=['date', 'time', 'path'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "indices = df1.path.str.contains(\"^L8/\\d{3}/\\d{3}/LC8\\d{13}[A-Z]{3}\\d{2}/LC8\\d{13}[A-Z]{3}\\d{2}_B4.TIF\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df1 = df1[indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df1['datetime'] = pd.to_datetime(df1.date + 'T' + df1.time, utc=False)\n",
    "df1.loc[:, 'entityId'] = df1.loc[:, 'path'].str[11:32]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df1.drop('path', axis=1, inplace=True)\n",
    "df1.drop('date', axis=1, inplace=True)\n",
    "df1.drop('time', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('l8-201512.geojson') as f:\n",
    "    data = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_filtered = [\n",
    "    { k: v for k, v in d['properties'].iteritems() if k in ['entityId', 'modifiedDate'] } for d in data['features']\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df2 = pd.DataFrame.from_dict(data_filtered)\n",
    "df2['modifiedDate'] = pd.to_datetime(df2.modifiedDate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.merge(df1, df2, on='entityId')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.datetime = df.datetime.apply(lambda dt: dt.tz_localize('US/Pacific').tz_convert('UTC'))\n",
    "df.modifiedDate = df.modifiedDate.apply(lambda dt: dt.tz_localize('UTC'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['delta'] = df.datetime - df.modifiedDate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count                     13020\n",
       "mean     0 days 17:48:27.439631\n",
       "std      0 days 02:03:31.482550\n",
       "min           -4 days +13:15:46\n",
       "25%             0 days 16:48:17\n",
       "50%             0 days 17:29:04\n",
       "75%             0 days 18:10:08\n",
       "max             1 days 02:11:16\n",
       "Name: delta, dtype: object"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.delta.describe()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}