Aggregation Job

Aggregation Job.ipynb

{
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  },
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from mozaggregator.aggregator import aggregate_metrics\n",
      "from mozaggregator.db import submit_aggregates\n",
      "\n",
      "from datetime import datetime, timedelta"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "yesterday = (datetime.now() - timedelta(days=1)).strftime(\"%Y%m%d\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%time \n",
      "aggregates = aggregate_metrics(sc, (\"nightly\", \"aurora\", \"beta\"), yesterday)\n",
      "print aggregates[0].count(), aggregates[1].count()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "4974 "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "570\n",
        "CPU times: user 28min 42s, sys: 33.7 s, total: 29min 16s\n",
        "Wall time: 6h 2min 22s\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%time submit_aggregates(aggregates)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "CPU times: user 7min 3s, sys: 16.7 s, total: 7min 20s\n",
        "Wall time: 1h 53min 43s\n"
       ]
      },
      {
       "output_type": "pyout",
       "prompt_number": 4,
       "text": [
        "(235, 10)"
       ]
      }
     ],
     "prompt_number": 4
    }
   ],
   "metadata": {}
  }
 ]
}