e10s-stability-analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import IPython\n",
    "\n",
    "from moztelemetry.spark import get_pings, get_pings_properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: pylab import has clobbered these variables: ['add']\n",
      "`%matplotlib` prevents importing * from pylab and numpy\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "IPython.core.pylabtools.figsize(16, 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "PING_OPTIONS = {\n",
    "    \"app\": \"Firefox\",\n",
    "    \"channel\": \"beta\",\n",
    "    \"version\": \"45.0\",\n",
    "    \"build_id\": \"*\",\n",
    "    \"submission_date\": (\"20160211\", \"20160218\"),\n",
    "    \"fraction\": 0.01,\n",
    "}\n",
    "main_pings = get_pings(sc, doc_type=\"main\", **PING_OPTIONS)\n",
    "main_ping_data = get_pings_properties(main_pings,\n",
    "                                      [\"environment/addons/activeExperiment/id\",\n",
    "                                       \"environment/addons/activeExperiment/branch\",\n",
    "                                       \"environment/settings/e10sEnabled\",\n",
    "                                       \"payload/info/subsessionLength\",\n",
    "                                       \"payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/content\",\n",
    "                                       \"payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/plugin\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "crash_pings = get_pings(sc, doc_type=\"crash\", **PING_OPTIONS)\n",
    "crash_ping_data = get_pings_properties(crash_pings,\n",
    "                                       [\"environment/addons/activeExperiment/id\",\n",
    "                                        \"environment/addons/activeExperiment/branch\",\n",
    "                                        \"environment/settings/e10sEnabled\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def experiment_filter(d):\n",
    "    try:\n",
    "        return d[\"environment/addons/activeExperiment/id\"] == \"e10s-beta45-withoutaddons@experiments.mozilla.org\" \\\n",
    "            and d[\"environment/addons/activeExperiment/branch\"] in (\"control-no-addons\", \"experiment-no-addons\")\n",
    "    except KeyError:\n",
    "        raise ValueError(\"Whoa nellie, missing a key: \" + repr(d))\n",
    "\n",
    "main_data_filtered = main_ping_data.filter(experiment_filter).cache()\n",
    "crash_data_filtered = crash_ping_data.filter(experiment_filter).cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(int,\n",
       "            {(u'control-no-addons', False): 20970,\n",
       "             (u'experiment-no-addons', True): 18867})"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from operator import add\n",
    "main_data_filtered.map(lambda d: (d[\"environment/addons/activeExperiment/branch\"],\n",
    "                                  d[\"environment/settings/e10sEnabled\"])).countByValue()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class Accumulators():\n",
    "    def __init__(self, cx):\n",
    "        self.main_crashes = cx.accumulator(0)\n",
    "        self.content_crashes = cx.accumulator(0)\n",
    "        self.plugin_crashes = cx.accumulator(0)\n",
    "        self.session_seconds = cx.accumulator(0)\n",
    "        \n",
    "    def hours(self):\n",
    "        return self.session_seconds.value / 360.0 / 1000\n",
    "    \n",
    "\n",
    "non_e10s = Accumulators(sc)\n",
    "e10s = Accumulators(sc)\n",
    "\n",
    "def process_main(d):\n",
    "    if d[\"environment/addons/activeExperiment/branch\"] == \"control-no-addons\":\n",
    "        acc = non_e10s\n",
    "    else:\n",
    "        acc = e10s\n",
    "\n",
    "    content = d[\"payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/content\"]\n",
    "    if content is not None:\n",
    "        acc.content_crashes.add(content)\n",
    "\n",
    "    plugin = d[\"payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/plugin\"]\n",
    "    if plugin is not None:\n",
    "        acc.plugin_crashes.add(plugin)\n",
    "        \n",
    "    acc.session_seconds.add(d[\"payload/info/subsessionLength\"])\n",
    "\n",
    "main_data_filtered.foreach(process_main)\n",
    "\n",
    "def process_crash(d):\n",
    "    if d[\"environment/addons/activeExperiment/branch\"] == \"control-no-addons\":\n",
    "        acc = non_e10s\n",
    "    else:\n",
    "        acc = e10s\n",
    "\n",
    "    acc.main_crashes.add(1)\n",
    "crash_data_filtered.foreach(process_crash)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                 non-e10s       e10s\n",
      "usage hours                           308        278\n",
      "chrome crashes                        357        174\n",
      "content crashes                        80        512\n",
      "plugin crashes                        252        186\n",
      "main crash rate                      1.16       0.63\n",
      "main+content crash rate              1.42       2.47\n",
      "plugin crash rate                    0.82       0.67\n"
     ]
    }
   ],
   "source": [
    "def p(name, type, v1, v2):\n",
    "    print (\"{:30s} {:>10\" + type + \"} {:>10\" + type + \"}\").format(name, v1, v2)\n",
    "\n",
    "p(\"\", \"s\", \"non-e10s\", \"e10s\")\n",
    "p(\"usage hours\", \".0f\", non_e10s.hours(), e10s.hours())\n",
    "p(\"chrome crashes\", \"d\", non_e10s.main_crashes.value, e10s.main_crashes.value)\n",
    "p(\"content crashes\", \"d\", non_e10s.content_crashes.value, e10s.content_crashes.value)\n",
    "p(\"plugin crashes\", \"d\", non_e10s.plugin_crashes.value, e10s.plugin_crashes.value)\n",
    "p(\"main crash rate\", \".2f\",\n",
    "  non_e10s.main_crashes.value / non_e10s.hours(),\n",
    "  e10s.main_crashes.value / e10s.hours())\n",
    "p(\"main+content crash rate\", \".2f\",\n",
    "  (non_e10s.main_crashes.value + non_e10s.content_crashes.value) / non_e10s.hours(),\n",
    "  (e10s.main_crashes.value + e10s.content_crashes.value) / e10s.hours())\n",
    "p(\"plugin crash rate\", \".2f\",\n",
    "  non_e10s.plugin_crashes.value / non_e10s.hours(),\n",
    "  e10s.plugin_crashes.value / e10s.hours())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}