Created
July 21, 2015 16:46
-
-
Save saptarshiguha/ad928472447a5cb20ca0 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| {"nbformat_minor": 0, "cells": [{"execution_count": 3, "cell_type": "code", "source": "import ujson as json\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nimport plotly.plotly as py\nfrom moztelemetry import get_pings, get_pings_properties, get_one_ping_per_client, get_clients_history\n%pylab inline\n## see bug ", "outputs": [{"output_type": "stream", "name": "stdout", "text": "Populating the interactive namespace from numpy and matplotlib\n"}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 4, "cell_type": "code", "source": "import datetime\nff39release = datetime.datetime(2015, 6, 30, 0,0, 1)\ndate_list = [ (ff39release + datetime.timedelta(days=x)).strftime(\"%Y%m%d\") for x in range(0, (datetime.datetime.today()-ff39release).days)]\ndef aggregate_arrays(xs, ys):\n if xs is None:\n return ys\n if ys is None:\n return xs\n return xs + ys", "outputs": [], "metadata": {"collapsed": false, "trusted": true}}, {"source": "I do not know of a way to extract submission date from within the telemetry v2 payload. Hence this loop.", "cell_type": "markdown", "metadata": {}}, {"execution_count": 5, "cell_type": "code", "source": "hcounts = { \"histograms/LOOP_SHARING_STATE_CHANGE_1\" : {}, \"histograms/LOOP_TWO_WAY_MEDIA_CONN_LENGTH_1\" : {} }\nharray = hcounts.keys()\nfrac = 0.1\nfor subdate in date_list:\n print(\"getting data for \"+subdate)\n pings = get_pings(sc, app=\"Firefox\", channel=\"release\", version=\"39.0\",submission_date=subdate, fraction=frac, schema=\"v2\")\n histograms = get_pings_properties(pings,harray, with_processes=False)\n print(\"getting ping props for \"+subdate)\n for hi in harray:\n print(\"filtering and counting \"+subdate)\n nonone1 = histograms.filter(lambda p: p[hi] is not None)\n x1 = nonone1.count() / frac\n aggregate = histograms.map(lambda p: p[hi]).reduce(aggregate_arrays) / frac\n hcounts[hi][subdate] = {'count':x1,'histo':aggregate}\n", "outputs": [{"output_type": "stream", "name": "stdout", "text": "getting data for 20150630\ngetting ping props for 20150630\nfiltering and counting 20150630\nfiltering and counting 20150630\ngetting data for 20150701\ngetting ping props for 20150701\nfiltering and counting 20150701\nfiltering and counting 20150701\ngetting data for 20150702\ngetting ping props for 20150702\nfiltering and counting 20150702\nfiltering and counting 20150702\ngetting data for 20150703\ngetting ping props for 20150703\nfiltering and counting 20150703\nfiltering and counting 20150703\ngetting data for 20150704\ngetting ping props for 20150704\nfiltering and counting 20150704\nfiltering and counting 20150704\ngetting data for 20150705\ngetting ping props for 20150705\nfiltering and counting 20150705\nfiltering and counting 20150705\ngetting data for 20150706\ngetting ping props for 20150706\nfiltering and counting 20150706\nfiltering and counting 20150706\ngetting data for 20150707\ngetting ping props for 20150707\nfiltering and counting 20150707\nfiltering and counting 20150707\ngetting data for 20150708\ngetting ping props for 20150708\nfiltering and counting 20150708\nfiltering and counting 20150708\ngetting data for 20150709\ngetting ping props for 20150709\nfiltering and counting 20150709\nfiltering and counting 20150709\ngetting data for 20150710\ngetting ping props for 20150710\nfiltering and counting 20150710\nfiltering and counting 20150710\ngetting data for 20150711\ngetting ping props for 20150711\nfiltering and counting 20150711\nfiltering and counting 20150711\ngetting data for 20150712\ngetting ping props for 20150712\nfiltering and counting 20150712\nfiltering and counting 20150712\ngetting data for 20150713\ngetting ping props for 20150713\nfiltering and counting 20150713\nfiltering and counting 20150713\ngetting data for 20150714\ngetting ping props for 20150714\nfiltering and counting 20150714\nfiltering and counting 20150714\ngetting data for 20150715\ngetting ping props for 20150715\nfiltering and counting 20150715\nfiltering and counting 20150715\ngetting data for 20150716\ngetting ping props for 20150716\nfiltering and counting 20150716\nfiltering and counting 20150716\ngetting data for 20150717\ngetting ping props for 20150717\nfiltering and counting 20150717\nfiltering and counting 20150717\ngetting data for 20150718\ngetting ping props for 20150718\nfiltering and counting 20150718\nfiltering and counting 20150718\ngetting data for 20150719\ngetting ping props for 20150719\nfiltering and counting 20150719\nfiltering and counting 20150719\ngetting data for 20150720\ngetting ping props for 20150720\nfiltering and counting 20150720\nfiltering and counting 20150720\n"}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 11, "cell_type": "code", "source": "try:\n import cPickle as pickle\nexcept:\n import pickle", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 13, "cell_type": "code", "source": "with open('/home/hadoop/bug117711.pickle', 'wb') as handle:\n pickle.dump(hcounts, handle)", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 22, "cell_type": "code", "source": "hcounts", "outputs": [{"execution_count": 22, "output_type": "execute_result", "data": {"text/plain": "{'histograms/LOOP_SHARING_STATE_CHANGE_1': {'20150630': {'count': 0.0,\n 'histo': None},\n '20150701': {'count': 0.0, 'histo': None},\n '20150702': {'count': 0.0, 'histo': None},\n '20150703': {'count': 10.0, 'histo': 0 10\n 1 10\n 2 0\n 3 0\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150704': {'count': 0.0, 'histo': None},\n '20150705': {'count': 0.0, 'histo': None},\n '20150706': {'count': 10.0, 'histo': 0 10\n 1 10\n 2 0\n 3 0\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150707': {'count': 10.0, 'histo': 0 0\n 1 0\n 2 20\n 3 20\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150708': {'count': 10.0, 'histo': 0 0\n 1 0\n 2 10\n 3 10\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150709': {'count': 10.0, 'histo': 0 10\n 1 10\n 2 0\n 3 0\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150710': {'count': 10.0, 'histo': 0 20\n 1 20\n 2 0\n 3 0\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150711': {'count': 0.0, 'histo': None},\n '20150712': {'count': 30.0, 'histo': 0 10\n 1 10\n 2 20\n 3 20\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150713': {'count': 10.0, 'histo': 0 10\n 1 10\n 2 10\n 3 10\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150714': {'count': 10.0, 'histo': 0 0\n 1 0\n 2 20\n 3 20\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150715': {'count': 20.0, 'histo': 0 0\n 1 0\n 2 20\n 3 20\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150716': {'count': 0.0, 'histo': None},\n '20150717': {'count': 10.0, 'histo': 0 0\n 1 0\n 2 50\n 3 50\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150718': {'count': 10.0, 'histo': 0 0\n 1 0\n 2 10\n 3 10\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150719': {'count': 20.0, 'histo': 0 10\n 1 10\n 2 20\n 3 20\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150720': {'count': 40.0, 'histo': 0 30\n 1 20\n 2 40\n 3 40\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64}},\n 'histograms/LOOP_TWO_WAY_MEDIA_CONN_LENGTH_1': {'20150630': {'count': 0.0,\n 'histo': None},\n '20150701': {'count': 0.0, 'histo': None},\n '20150702': {'count': 0.0, 'histo': None},\n '20150703': {'count': 0.0, 'histo': None},\n '20150704': {'count': 10.0, 'histo': 0 0\n 1 10\n 2 0\n 3 0\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150705': {'count': 0.0, 'histo': None},\n '20150706': {'count': 0.0, 'histo': None},\n '20150707': {'count': 50.0, 'histo': 0 0\n 1 10\n 2 30\n 3 50\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150708': {'count': 0.0, 'histo': None},\n '20150709': {'count': 0.0, 'histo': None},\n '20150710': {'count': 20.0, 'histo': 0 0\n 1 0\n 2 10\n 3 20\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150711': {'count': 20.0, 'histo': 0 0\n 1 0\n 2 10\n 3 10\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150712': {'count': 60.0, 'histo': 0 0\n 1 10\n 2 90\n 3 120\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150713': {'count': 30.0, 'histo': 0 0\n 1 10\n 2 10\n 3 10\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150714': {'count': 70.0, 'histo': 0 0\n 1 40\n 2 30\n 3 40\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150715': {'count': 10.0, 'histo': 0 0\n 1 10\n 2 0\n 3 0\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150716': {'count': 30.0, 'histo': 0 0\n 1 0\n 2 30\n 3 30\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150717': {'count': 30.0, 'histo': 0 0\n 1 0\n 2 10\n 3 30\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150718': {'count': 20.0, 'histo': 0 0\n 1 0\n 2 0\n 3 40\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150719': {'count': 30.0, 'histo': 0 0\n 1 10\n 2 10\n 3 20\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64},\n '20150720': {'count': 20.0, 'histo': 0 0\n 1 0\n 2 50\n 3 30\n 4 0\n 5 0\n 6 0\n 7 0\n 8 0\n dtype: float64}}}"}, "metadata": {}}], "metadata": {"scrolled": false, "collapsed": false, "trusted": true}}, {"execution_count": null, "cell_type": "code", "source": "", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}], "nbformat": 4, "metadata": {"kernelspec": {"display_name": "Python 2", "name": "python2", "language": "python"}, "language_info": {"mimetype": "text/x-python", "nbconvert_exporter": "python", "version": "2.7.9", "name": "python", "file_extension": ".py", "pygments_lexer": "ipython2", "codemirror_mode": {"version": 2, "name": "ipython"}}}} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment