e10s-stability-analysis

e10s-stability-analysis.ipynb

e10s-stability-analysis.py

# coding: utf-8

# ### e10s-beta45-withoutaddons: Stability analysis

# [Bug 1222890](https://bugzilla.mozilla.org/show_bug.cgi?id=1222890)

# This analysis compares e10s and non-e10s crash rates.

# In[2]:

import IPython

from moztelemetry.spark import get_pings, get_pings_properties


# In[3]:

get_ipython().magic(u'pylab inline')
IPython.core.pylabtools.figsize(16, 7)


# In[4]:

# This block contains all the things one should need to change for testing a new experiment/build/date range.
PING_OPTIONS = {
    "app": "Firefox",
    "channel": "beta",
    "version": "46.0",
    "build_id": "*",
    "submission_date": ("20160309", "20160321")
}

EXPERIMENT_ID = "e10s-beta46-noapz@experiments.mozilla.org"
EXPERIMENT_BRANCHES = ("control-no-addons", "experiment-no-addons")


# In[5]:

main_pings = get_pings(sc, doc_type="main", **PING_OPTIONS)
main_ping_data = get_pings_properties(main_pings,
                                      ["environment/addons/activeExperiment/id",
                                       "environment/addons/activeExperiment/branch",
                                       "environment/settings/e10sEnabled",
                                       "payload/info/subsessionLength",
                                       "payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/content",
                                       "payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/plugin"],
                                      with_processes=True)


# In[6]:

crash_pings = get_pings(sc, doc_type="crash", **PING_OPTIONS)
crash_ping_data = get_pings_properties(crash_pings,
                                       ["environment/addons/activeExperiment/id",
                                        "environment/addons/activeExperiment/branch",
                                        "environment/settings/e10sEnabled"])


# In[7]:

def experiment_filter(d):
    try:
        return d["environment/addons/activeExperiment/id"] == EXPERIMENT_ID             and d["environment/addons/activeExperiment/branch"] in EXPERIMENT_BRANCHES
    except KeyError:
        raise ValueError("Whoa nellie, missing a key: " + repr(d))

main_data_filtered = main_ping_data.filter(experiment_filter).cache()
crash_data_filtered = crash_ping_data.filter(experiment_filter).cache()


# How many pings do we have in each branch?

# In[8]:

from operator import add
main_data_filtered.map(lambda d: (d["environment/addons/activeExperiment/branch"],
                                  d["environment/settings/e10sEnabled"])).countByValue()


# In[9]:

class Accumulators():
    def __init__(self, cx):
        self.main_crashes = cx.accumulator(0)
        self.content_crashes = cx.accumulator(0)
        self.plugin_crashes = cx.accumulator(0)
        self.session_seconds = cx.accumulator(0)
        
    def hours(self):
        return self.session_seconds.value / 3600.0 / 1000

non_e10s = Accumulators(sc)
e10s = Accumulators(sc)


# In[10]:

def process_main(d):
    if d["environment/addons/activeExperiment/branch"] == "control-no-addons":
        acc = non_e10s
    else:
        acc = e10s

    content = d["payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/content_parent"]
    if content is not None:
        acc.content_crashes.add(content)

    plugin = d["payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/plugin_parent"]
    if plugin is not None:
        acc.plugin_crashes.add(plugin)
    
    length = d["payload/info/subsessionLength"]
    if length is not None:
        acc.session_seconds.add(length)

main_data_filtered.foreach(process_main)


# In[11]:

def process_crash(d):
    if d["environment/addons/activeExperiment/branch"] == "control-no-addons":
        acc = non_e10s
    else:
        acc = e10s

    acc.main_crashes.add(1)

crash_data_filtered.foreach(process_crash)


# In[12]:

def p(name, type, v1, v2):
    print ("{:30s} {:>10" + type + "} {:>10" + type + "}").format(name, v1, v2)

p("", "s", "non-e10s", "e10s")
p("usage hours", ".0f", non_e10s.hours(), e10s.hours())
p("chrome crashes", "d", non_e10s.main_crashes.value, e10s.main_crashes.value)
p("content crashes", "d", non_e10s.content_crashes.value, e10s.content_crashes.value)
p("plugin crashes", "d", non_e10s.plugin_crashes.value, e10s.plugin_crashes.value)
p("main crash rate", ".2f",
  non_e10s.main_crashes.value / non_e10s.hours(),
  e10s.main_crashes.value / e10s.hours())
p("main+content crash rate", ".2f",
  (non_e10s.main_crashes.value + non_e10s.content_crashes.value) / non_e10s.hours(),
  (e10s.main_crashes.value + e10s.content_crashes.value) / e10s.hours())
p("plugin crash rate", ".2f",
  non_e10s.plugin_crashes.value / non_e10s.hours(),
  e10s.plugin_crashes.value / e10s.hours())