Sum ARG25 Tile Sizes

SumTileSizes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import dbf\n",
    "import glob\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BASEDIR = '/g/data/rs0/tiles/EPSG4326_1deg_0.00025pixel/'\n",
    "SATS = ('LS5_TM',  'LS7_ETM',  'LS8_OLI',  'LS8_OLI_TIRS')\n",
    "\n",
    "def counts_from_dbf(filename):\n",
    "    table = dbf.Table(filename)\n",
    "    table.open()\n",
    "    result = [(record.x, record.y) for record in table]\n",
    "    table.close()\n",
    "    return result\n",
    "\n",
    "def sizeof_fmt(num, suffix='B'):\n",
    "    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:\n",
    "        if abs(num) < 1024.0:\n",
    "            return \"%3.2f%s%s\" % (num, unit, suffix)\n",
    "        num /= 1024.0\n",
    "    return \"%.1f%s%s\" % (num, 'Yi', suffix)\n",
    "\n",
    "def calc_tiles_size(path, tile_indexes):\n",
    "    totalsize = 0\n",
    "    filecount = 0\n",
    "    for x, y in tile_indexes:\n",
    "        globstr = '{path}/{x:03d}_{y:04d}/*/*NBAR*.tif'.format(path=path, x=x, y=y) # All y's are negative, so pad to 4\n",
    "        for filename in glob.glob(globstr):\n",
    "            filesize = os.path.getsize(filename)\n",
    "            totalsize += filesize\n",
    "            filecount += 1\n",
    "    \n",
    "    return totalsize, filecount\n",
    "\n",
    "\n",
    "def tile_size_summaries(tile_indexes):\n",
    "    summary = {}\n",
    "    total_size = 0\n",
    "    total_count = 0\n",
    "    for sat in SATS:\n",
    "        size, count = calc_tiles_size(BASEDIR + sat, tile_indexes)\n",
    "        summary[sat] = {'filesize': size, 'filecount': count}\n",
    "        total_size += size\n",
    "        total_count += count\n",
    "    \n",
    "    summary['total'] = {'filesize': total_size, 'filecount': total_count}\n",
    "    \n",
    "    return summary\n",
    "\n",
    "\n",
    "\n",
    "def print_sizes(sat, size, count):\n",
    "    print('{sat:15}: filesize: {size}, filecount: {count}'.format(\n",
    "            sat=sat, size=sizeof_fmt(size), count=count))\n",
    "    \n",
    "\n",
    "def print_summary(summary):\n",
    "    total = summary['total']\n",
    "    del summary['total']\n",
    "    for sat, counts in summary.items():\n",
    "        print_sizes(sat, counts['filesize'], counts['filecount'])\n",
    "    print_sizes('Total', total['filesize'], total['filecount'])\n",
    "    summary['total'] = total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "South Australia\n",
      "LS5_TM         : filesize: 10.11TiB, filecount: 93288\n",
      "LS7_ETM        : filesize: 11.04TiB, filecount: 113640\n",
      "LS8_OLI        : filesize: 27.30GiB, filecount: 59\n",
      "LS8_OLI_TIRS   : filesize: 9.62TiB, filecount: 18780\n",
      "Total          : filesize: 30.79TiB, filecount: 225767\n",
      "\n",
      "Victoria\n",
      "LS5_TM         : filesize: 3.52TiB, filecount: 31140\n",
      "LS7_ETM        : filesize: 3.97TiB, filecount: 38228\n",
      "LS8_OLI        : filesize: 6.11GiB, filecount: 9\n",
      "LS8_OLI_TIRS   : filesize: 3.27TiB, filecount: 6451\n",
      "Total          : filesize: 10.76TiB, filecount: 75828\n"
     ]
    }
   ],
   "source": [
    "SA_tiles = counts_from_dbf('SA_tiles')\n",
    "VIC_tiles = counts_from_dbf('VIC_tiles2')\n",
    "\n",
    "sa_summary = tile_size_summaries(SA_tiles)\n",
    "print \"South Australia\"\n",
    "print_summary(sa_summary)\n",
    "print('')\n",
    "\n",
    "\n",
    "vic_summary = tile_size_summaries(VIC_tiles)\n",
    "print \"Victoria\"\n",
    "print_summary(vic_summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'LS5_TM': {'filecount': 93288, 'filesize': 11113512974506},\n",
       " 'LS7_ETM': {'filecount': 113640, 'filesize': 12136130058079},\n",
       " 'LS8_OLI': {'filecount': 59, 'filesize': 29314525815},\n",
       " 'LS8_OLI_TIRS': {'filecount': 18780, 'filesize': 10576118640562},\n",
       " 'total': {'filecount': 225767, 'filesize': 33855076198962}}"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sa_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'LS5_TM': {'filecount': 31140, 'filesize': 3867227309242},\n",
       " 'LS7_ETM': {'filecount': 38228, 'filesize': 4364210804121},\n",
       " 'LS8_OLI': {'filecount': 9, 'filesize': 6565189905},\n",
       " 'LS8_OLI_TIRS': {'filecount': 6451, 'filesize': 3592859610581},\n",
       " 'total': {'filecount': 75828, 'filesize': 11830862913849}}"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vic_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}