e10s-stability-analysis

e10s-stability-analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### e10s-beta45-withoutaddons: Stability analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Bug 1222890](https://bugzilla.mozilla.org/show_bug.cgi?id=1222890)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This analysis compares e10s and non-e10s crash rates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unable to parse whitelist (/home/hadoop/anaconda2/lib/python2.7/site-packages/moztelemetry/bucket-whitelist.json). Assuming all histograms are acceptable.\n"
     ]
    }
   ],
   "source": [
    "import IPython\n",
    "\n",
    "from moztelemetry.spark import get_pings, get_pings_properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hadoop/anaconda2/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "IPython.core.pylabtools.figsize(16, 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This block contains all the things one should need to change for testing a new experiment/build/date range.\n",
    "PING_OPTIONS = {\n",
    "    \"app\": \"Firefox\",\n",
    "    \"channel\": \"beta\",\n",
    "    \"version\": \"46.0\",\n",
    "    \"build_id\": \"*\",\n",
    "    \"submission_date\": (\"20160309\", \"20160321\")\n",
    "}\n",
    "\n",
    "EXPERIMENT_ID = \"e10s-beta46-noapz@experiments.mozilla.org\"\n",
    "EXPERIMENT_BRANCHES = (\"control-no-addons\", \"experiment-no-addons\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "main_pings = get_pings(sc, doc_type=\"main\", **PING_OPTIONS)\n",
    "main_ping_data = get_pings_properties(main_pings,\n",
    "                                      [\"environment/addons/activeExperiment/id\",\n",
    "                                       \"environment/addons/activeExperiment/branch\",\n",
    "                                       \"environment/settings/e10sEnabled\",\n",
    "                                       \"payload/info/subsessionLength\",\n",
    "                                       \"payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/content\",\n",
    "                                       \"payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/plugin\"],\n",
    "                                      with_processes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "crash_pings = get_pings(sc, doc_type=\"crash\", **PING_OPTIONS)\n",
    "crash_ping_data = get_pings_properties(crash_pings,\n",
    "                                       [\"environment/addons/activeExperiment/id\",\n",
    "                                        \"environment/addons/activeExperiment/branch\",\n",
    "                                        \"environment/settings/e10sEnabled\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def experiment_filter(d):\n",
    "    try:\n",
    "        return d[\"environment/addons/activeExperiment/id\"] == EXPERIMENT_ID \\\n",
    "            and d[\"environment/addons/activeExperiment/branch\"] in EXPERIMENT_BRANCHES\n",
    "    except KeyError:\n",
    "        raise ValueError(\"Whoa nellie, missing a key: \" + repr(d))\n",
    "\n",
    "main_data_filtered = main_ping_data.filter(experiment_filter).cache()\n",
    "crash_data_filtered = crash_ping_data.filter(experiment_filter).cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How many pings do we have in each branch?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(int,\n",
       "            {(u'control-no-addons', False): 2098989,\n",
       "             (u'experiment-no-addons', False): 12,\n",
       "             (u'experiment-no-addons', True): 1893239})"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from operator import add\n",
    "main_data_filtered.map(lambda d: (d[\"environment/addons/activeExperiment/branch\"],\n",
    "                                  d[\"environment/settings/e10sEnabled\"])).countByValue()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class Accumulators():\n",
    "    def __init__(self, cx):\n",
    "        self.main_crashes = cx.accumulator(0)\n",
    "        self.content_crashes = cx.accumulator(0)\n",
    "        self.plugin_crashes = cx.accumulator(0)\n",
    "        self.session_seconds = cx.accumulator(0)\n",
    "        \n",
    "    def hours(self):\n",
    "        return self.session_seconds.value / 3600.0 / 1000\n",
    "\n",
    "non_e10s = Accumulators(sc)\n",
    "e10s = Accumulators(sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_main(d):\n",
    "    if d[\"environment/addons/activeExperiment/branch\"] == \"control-no-addons\":\n",
    "        acc = non_e10s\n",
    "    else:\n",
    "        acc = e10s\n",
    "\n",
    "    content = d[\"payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/content_parent\"]\n",
    "    if content is not None:\n",
    "        acc.content_crashes.add(content)\n",
    "\n",
    "    plugin = d[\"payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/plugin_parent\"]\n",
    "    if plugin is not None:\n",
    "        acc.plugin_crashes.add(plugin)\n",
    "    \n",
    "    length = d[\"payload/info/subsessionLength\"]\n",
    "    if length is not None:\n",
    "        acc.session_seconds.add(length)\n",
    "\n",
    "main_data_filtered.foreach(process_main)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_crash(d):\n",
    "    if d[\"environment/addons/activeExperiment/branch\"] == \"control-no-addons\":\n",
    "        acc = non_e10s\n",
    "    else:\n",
    "        acc = e10s\n",
    "\n",
    "    acc.main_crashes.add(1)\n",
    "\n",
    "crash_data_filtered.foreach(process_crash)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                 non-e10s       e10s\n",
      "usage hours                          3312       2547\n",
      "chrome crashes                      50409      20203\n",
      "content crashes                      6320      50546\n",
      "plugin crashes                      26843      30741\n",
      "main crash rate                     15.22       7.93\n",
      "main+content crash rate             17.13      27.77\n",
      "plugin crash rate                    8.10      12.07\n"
     ]
    }
   ],
   "source": [
    "def p(name, type, v1, v2):\n",
    "    print (\"{:30s} {:>10\" + type + \"} {:>10\" + type + \"}\").format(name, v1, v2)\n",
    "\n",
    "p(\"\", \"s\", \"non-e10s\", \"e10s\")\n",
    "p(\"usage hours\", \".0f\", non_e10s.hours(), e10s.hours())\n",
    "p(\"chrome crashes\", \"d\", non_e10s.main_crashes.value, e10s.main_crashes.value)\n",
    "p(\"content crashes\", \"d\", non_e10s.content_crashes.value, e10s.content_crashes.value)\n",
    "p(\"plugin crashes\", \"d\", non_e10s.plugin_crashes.value, e10s.plugin_crashes.value)\n",
    "p(\"main crash rate\", \".2f\",\n",
    "  non_e10s.main_crashes.value / non_e10s.hours(),\n",
    "  e10s.main_crashes.value / e10s.hours())\n",
    "p(\"main+content crash rate\", \".2f\",\n",
    "  (non_e10s.main_crashes.value + non_e10s.content_crashes.value) / non_e10s.hours(),\n",
    "  (e10s.main_crashes.value + e10s.content_crashes.value) / e10s.hours())\n",
    "p(\"plugin crash rate\", \".2f\",\n",
    "  non_e10s.plugin_crashes.value / non_e10s.hours(),\n",
    "  e10s.plugin_crashes.value / e10s.hours())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}

e10s-stability-analysis.py

# coding: utf-8

# ### e10s-beta45-withoutaddons: Stability analysis

# [Bug 1222890](https://bugzilla.mozilla.org/show_bug.cgi?id=1222890)

# This analysis compares e10s and non-e10s crash rates.

# In[2]:

import IPython

from moztelemetry.spark import get_pings, get_pings_properties


# In[3]:

get_ipython().magic(u'pylab inline')
IPython.core.pylabtools.figsize(16, 7)


# In[4]:

# This block contains all the things one should need to change for testing a new experiment/build/date range.
PING_OPTIONS = {
    "app": "Firefox",
    "channel": "beta",
    "version": "46.0",
    "build_id": "*",
    "submission_date": ("20160309", "20160321")
}

EXPERIMENT_ID = "e10s-beta46-noapz@experiments.mozilla.org"
EXPERIMENT_BRANCHES = ("control-no-addons", "experiment-no-addons")


# In[5]:

main_pings = get_pings(sc, doc_type="main", **PING_OPTIONS)
main_ping_data = get_pings_properties(main_pings,
                                      ["environment/addons/activeExperiment/id",
                                       "environment/addons/activeExperiment/branch",
                                       "environment/settings/e10sEnabled",
                                       "payload/info/subsessionLength",
                                       "payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/content",
                                       "payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/plugin"],
                                      with_processes=True)


# In[6]:

crash_pings = get_pings(sc, doc_type="crash", **PING_OPTIONS)
crash_ping_data = get_pings_properties(crash_pings,
                                       ["environment/addons/activeExperiment/id",
                                        "environment/addons/activeExperiment/branch",
                                        "environment/settings/e10sEnabled"])


# In[7]:

def experiment_filter(d):
    try:
        return d["environment/addons/activeExperiment/id"] == EXPERIMENT_ID             and d["environment/addons/activeExperiment/branch"] in EXPERIMENT_BRANCHES
    except KeyError:
        raise ValueError("Whoa nellie, missing a key: " + repr(d))

main_data_filtered = main_ping_data.filter(experiment_filter).cache()
crash_data_filtered = crash_ping_data.filter(experiment_filter).cache()


# How many pings do we have in each branch?

# In[8]:

from operator import add
main_data_filtered.map(lambda d: (d["environment/addons/activeExperiment/branch"],
                                  d["environment/settings/e10sEnabled"])).countByValue()


# In[9]:

class Accumulators():
    def __init__(self, cx):
        self.main_crashes = cx.accumulator(0)
        self.content_crashes = cx.accumulator(0)
        self.plugin_crashes = cx.accumulator(0)
        self.session_seconds = cx.accumulator(0)
        
    def hours(self):
        return self.session_seconds.value / 3600.0 / 1000

non_e10s = Accumulators(sc)
e10s = Accumulators(sc)


# In[10]:

def process_main(d):
    if d["environment/addons/activeExperiment/branch"] == "control-no-addons":
        acc = non_e10s
    else:
        acc = e10s

    content = d["payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/content_parent"]
    if content is not None:
        acc.content_crashes.add(content)

    plugin = d["payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/plugin_parent"]
    if plugin is not None:
        acc.plugin_crashes.add(plugin)
    
    length = d["payload/info/subsessionLength"]
    if length is not None:
        acc.session_seconds.add(length)

main_data_filtered.foreach(process_main)


# In[11]:

def process_crash(d):
    if d["environment/addons/activeExperiment/branch"] == "control-no-addons":
        acc = non_e10s
    else:
        acc = e10s

    acc.main_crashes.add(1)

crash_data_filtered.foreach(process_crash)


# In[12]:

def p(name, type, v1, v2):
    print ("{:30s} {:>10" + type + "} {:>10" + type + "}").format(name, v1, v2)

p("", "s", "non-e10s", "e10s")
p("usage hours", ".0f", non_e10s.hours(), e10s.hours())
p("chrome crashes", "d", non_e10s.main_crashes.value, e10s.main_crashes.value)
p("content crashes", "d", non_e10s.content_crashes.value, e10s.content_crashes.value)
p("plugin crashes", "d", non_e10s.plugin_crashes.value, e10s.plugin_crashes.value)
p("main crash rate", ".2f",
  non_e10s.main_crashes.value / non_e10s.hours(),
  e10s.main_crashes.value / e10s.hours())
p("main+content crash rate", ".2f",
  (non_e10s.main_crashes.value + non_e10s.content_crashes.value) / non_e10s.hours(),
  (e10s.main_crashes.value + e10s.content_crashes.value) / e10s.hours())
p("plugin crash rate", ".2f",
  non_e10s.plugin_crashes.value / non_e10s.hours(),
  e10s.plugin_crashes.value / e10s.hours())