plugin-block-experiment

plugin-block-experiment.ipynb

plugin-block-experiment.py

# coding: utf-8

# # Plugin Block Experiment

# ## STATUS: UNREVIEWED

# In[1]:

import ujson as json
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import IPython

from __future__ import division
from montecarlino import grouped_permutation_test

from moztelemetry.spark import get_pings, get_pings_properties

get_ipython().magic(u'pylab inline')
IPython.core.pylabtools.figsize(16, 7)


# In[2]:

sc.defaultParallelism


# In[6]:

pings = get_pings(sc, app="Firefox", channel="beta", version="47.0",
                  submission_date=("20160510", "20160517"),
                  build_id=("20160510000000", "20160517999999"), doc_type="main")


# In[10]:

properties = [
    "clientId",
    "environment/addons/activeExperiment/id",
    "environment/addons/activeExperiment/branch",
    "payload/info/subsessionLength",
    "payload/keyedHistograms/BLOCKED_ON_PLUGIN_INSTANCE_INIT_MS",
    "payload/keyedHistograms/BLOCKED_ON_PLUGIN_INSTANCE_DESTROY_MS",
    "payload/histograms/PLUGIN_BLOCKED_FOR_STABILITY",
    "payload/histograms/INPUT_EVENT_RESPONSE_MS",
    "payload/histograms/FX_PAGE_LOAD_MS",
    "payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/plugin",
    "payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/pluginhang",
    "payload/histograms/PLUGIN_HANG_TIME",
]
dataset = get_pings_properties(pings, properties, with_processes=True)
def is_in_experiment(ping):
    return ping["environment/addons/activeExperiment/id"] == "plugin-block-beta47@experiments.mozilla.org" and         ping["environment/addons/activeExperiment/branch"] is not None
    
filtered = dataset.filter(is_in_experiment)


# In[12]:

cached = filtered.cache()


# In[23]:

f = cached.first()
f


# In[17]:

# Compare the total usage hours across cohorts
from operator import add
cached.map(lambda ping: (ping["environment/addons/activeExperiment/branch"], ping["payload/info/subsessionLength"] or 0)).foldByKey(0, add).collect()


# In[36]:

# How many plugin instances did we block?
blocks_by_branch = cached.map(lambda ping: (ping["environment/addons/activeExperiment/branch"], ping["payload/histograms/PLUGIN_BLOCKED_FOR_STABILITY"] or 0)).foldByKey(0, add).collectAsMap()
blocks_by_branch


# In[37]:

# How many plugin instances did we load?
def map_plugin_instances(ping):
    c = 0
    v = ping["payload/keyedHistograms/BLOCKED_ON_PLUGIN_INSTANCE_INIT_MS"]
    if v is not None:
        for name, data in v.iteritems():
            if name.startswith("Shockwave Flash") and name.endswith("_parent") and data is not None:
                c += sum(data)
    return (ping["environment/addons/activeExperiment/branch"], c)

loads_by_branch = cached.map(map_plugin_instances).foldByKey(0, add).collectAsMap()
loads_by_branch


# In[61]:

# What fraction of total plugin loads are we blocking?
"{:.2f}%".format(blocks_by_branch['aggressive'] / loads_by_branch['aggressive'] * 100)


# In[46]:

# How many plugin crashes?
crashes_by_branch = cached.map(lambda ping: (ping["environment/addons/activeExperiment/branch"], ping["payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/plugin"] or 0)).foldByKey(0, add).collectAsMap()
crashes_by_branch


# In[59]:

# TODO: How confident are we/can we be in the following statement?
print "The aggressive blocklist reduced the plugin crash rate by {:.2f}%".format(100 * (crashes_by_branch['control'] - crashes_by_branch['aggressive']) / crashes_by_branch['control'])


# In[48]:

# How many plugin hangs?
hangs_by_branch = cached.map(lambda ping: (ping["environment/addons/activeExperiment/branch"], sum(ping["payload/histograms/PLUGIN_HANG_TIME"]) or 0)).foldByKey(0, add).collectAsMap()
hangs_by_branch


# In[60]:

# TODO: How confident are we/can we be in the following statement?
print "The aggressive blocklist reduced the plugin hang rate by {:0.2f}%".format(100 * (hangs_by_branch['control'] - hangs_by_branch['aggressive']) / hangs_by_branch['control'])