Bug 1274975 - Validate moving the clientId loading earlier during startup

Bug 1274975 - Validate moving the clientId loading earlier during startup.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hadoop/anaconda2/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unable to parse whitelist (/home/hadoop/anaconda2/lib/python2.7/site-packages/moztelemetry/histogram-whitelists.json). Assuming all histograms are acceptable.\n"
     ]
    }
   ],
   "source": [
    "import datetime as dt\n",
    "import matplotlib.pyplot as plt\n",
    "import ujson as json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import copy as cp\n",
    "\n",
    "from moztelemetry import get_pings, get_pings_properties, get_one_ping_per_client, get_clients_history"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How many crash pings with no client id the week before 1233986 landed?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "build_ids = (\"20160622000000\", \"20160628999999\")\n",
    "submission_dates = (\"20160622\", \"20160628\")\n",
    "crash_pings = get_pings(sc,\n",
    "                        app=\"Firefox\",\n",
    "                        channel=\"nightly\",\n",
    "                        build_id=build_ids,\n",
    "                        submission_date=submission_dates,\n",
    "                        doc_type=\"crash\",\n",
    "                        schema=\"v4\",\n",
    "                        fraction=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3621"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "crash_pings_count = crash_pings.count()\n",
    "crash_pings_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "subset = get_pings_properties(crash_pings, [\"meta/clientId\",\n",
    "                                            \"meta/submissionDate\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "crash_pings_no_clientid = subset.filter(lambda p: not p.get(\"meta/clientId\", None))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "80"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_clientid_count = crash_pings_no_clientid.count()\n",
    "no_clientid_count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How many crash pings with no client id the after 1233986 landed?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "build_ids = (\"20160629000000\", \"20160705030222\")\n",
    "submission_dates = (\"20160629\", \"20160706\")\n",
    "crash_pings = get_pings(sc,\n",
    "                        app=\"Firefox\",\n",
    "                        channel=\"nightly\",\n",
    "                        build_id=build_ids,\n",
    "                        submission_date=submission_dates,\n",
    "                        doc_type=\"crash\",\n",
    "                        schema=\"v4\",\n",
    "                        fraction=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3136"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "crash_pings_count = crash_pings.count()\n",
    "crash_pings_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "subset = get_pings_properties(crash_pings, [\"meta/clientId\",\n",
    "                                            \"meta/submissionDate\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "crash_pings_no_clientid = subset.filter(lambda p: not p.get(\"meta/clientId\", None))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_clientid_count = crash_pings_no_clientid.count()\n",
    "no_clientid_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}

Bug 1274975 - Validate moving the clientId loading earlier during startup.py

# coding: utf-8

# In[1]:

import datetime as dt
import matplotlib.pyplot as plt
import ujson as json
import pandas as pd
import numpy as np
import copy as cp

from moztelemetry import get_pings, get_pings_properties, get_one_ping_per_client, get_clients_history


# ### How many crash pings with no client id the week before 1233986 landed?

# In[13]:

build_ids = ("20160622000000", "20160628999999")
submission_dates = ("20160622", "20160628")
crash_pings = get_pings(sc,
                        app="Firefox",
                        channel="nightly",
                        build_id=build_ids,
                        submission_date=submission_dates,
                        doc_type="crash",
                        schema="v4",
                        fraction=1.0)


# In[14]:

crash_pings_count = crash_pings.count()
crash_pings_count


# In[15]:

subset = get_pings_properties(crash_pings, ["meta/clientId",
                                            "meta/submissionDate"])


# In[16]:

crash_pings_no_clientid = subset.filter(lambda p: not p.get("meta/clientId", None))


# In[17]:

no_clientid_count = crash_pings_no_clientid.count()
no_clientid_count


# ### How many crash pings with no client id the after 1233986 landed?

# In[18]:

build_ids = ("20160629000000", "20160705030222")
submission_dates = ("20160629", "20160706")
crash_pings = get_pings(sc,
                        app="Firefox",
                        channel="nightly",
                        build_id=build_ids,
                        submission_date=submission_dates,
                        doc_type="crash",
                        schema="v4",
                        fraction=1.0)


# In[19]:

crash_pings_count = crash_pings.count()
crash_pings_count


# In[20]:

subset = get_pings_properties(crash_pings, ["meta/clientId",
                                            "meta/submissionDate"])


# In[21]:

crash_pings_no_clientid = subset.filter(lambda p: not p.get("meta/clientId", None))


# In[22]:

no_clientid_count = crash_pings_no_clientid.count()
no_clientid_count


# In[ ]: