mdv2018-week06.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# gist -u https://gist.github.com/e89e674dda2a065143ffc138c3774a3c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gzip            # can uncompress gzipped files, useful for accessing the pleiades data\n",
    "import io              # useful routines for input/output\n",
    "import pandas as pd    # for working with \"rows/columns\" oriented data\n",
    "import urllib.request  # for loading documents using http\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "import matplotlib # plotting\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "pd.options.display.max_columns = 999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pd.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create DFs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sites = pd.DataFrame({'id':['s01','s02','s03','s04'],\n",
    "                      'type':['settlement','cave','temple','settlement']},\n",
    "                      columns = ['id','type'])\n",
    "# sites"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "finds = pd.DataFrame({'id':['f01','f02','f03'],\n",
    "                      's_id':['s01','s01','s02'],\n",
    "                      'period':['early','middle','late']},\n",
    "                      columns = ['id','s_id','period'])\n",
    "# finds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "site_types = pd.DataFrame({'id':['settlement','cave','temple'],\n",
    "                      'category':['habitation','uncertain','religious']},\n",
    "                      columns = ['id','category'])\n",
    "\n",
    "#site_types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "periods = pd.DataFrame({'id':['early','middle','late','classic'],\n",
    "                      'startdate':[100,250,725,350],\n",
    "                      'enddate':[250,725,850,550]})\n",
    "# periods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "s_to_p = pd.DataFrame({'s_id':['s01','s01','s02','s02','s03'],\n",
    "                      'p_id':['early','middle','late','middle','classic']})\n",
    "# s_to_p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Types of Relations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# one to one\n",
    "finds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# many to one\n",
    "sites.merge(site_types, left_on = 'type', right_on = 'id')\n",
    "\n",
    "#.groupby(by = 'category').count().plot.bar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# one to many\n",
    "sites.merge(finds, left_on = 'id', right_on = 's_id', how = 'left') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# many to many\n",
    "sites.merge(s_to_p, left_on = 'id', right_on = 's_id')\\\n",
    " .merge(periods, left_on = 'p_id', right_on = 'id')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Doing More"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# list finds after 250 on sites first occupied before 250\n",
    "finds_merged = finds.merge(sites, left_on = 's_id', right_on ='id', suffixes = ('','_sites'))\\\n",
    " .merge(periods, left_on = 'period', right_on = 'id', suffixes = ('','_periods'))\\\n",
    " .drop(columns = ['id_periods','id_sites'])\n",
    "\n",
    "earliest_dates = s_to_p.merge(periods, left_on = 'p_id', right_on = 'id')\\\n",
    " .groupby(by = 's_id')\\\n",
    " .min().reset_index(level = 's_id')[['s_id','startdate']]\n",
    "\n",
    "finds_merged.merge(earliest_dates, on = 's_id', suffixes = ('','_site'))\\\n",
    " .query('startdate_site < 250 and startdate >=250')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Working with GeoTiffs in Python"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is in part an exercise in installing the rasterio and imageio python module."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import rasterio\n",
    "import imageio\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "viz = rasterio.open('cumulative_intervis_cumulative.tiff') # or current name\n",
    "viz_array = viz.read(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "viz= imageio.imread('cumulative_intervis_cumulative.tiff')\n",
    "viz_array = np.asarray(viz)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "viz_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "viz_hist = np.histogram(viz_array, range(0,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "viz_hist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "viz_hist_df = pd.DataFrame(viz_hist[0], columns = ['vcount'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "viz_hist_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "viz_hist_df.query('vcount > 0 and vcount < 100000').plot.bar()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}