flash-experiment-nightly-2017

flash-experiment-nightly-2017.ipynb

flash-experiment-nightly-2017.py

# coding: utf-8

# In[17]:

import ujson as json
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import plotly.plotly as py

from plotly.graph_objs import *
from moztelemetry import get_pings_properties
from moztelemetry.dataset import Dataset
from moztelemetry.histogram import Histogram
from operator import add

from datetime import date, timedelta

get_ipython().magic(u'matplotlib inline')


# In[18]:

from collections import namedtuple


# In[19]:

pings = Dataset.from_source("telemetry")     .where(docType="main")     .where(submissionDate=lambda d: "20170413" <= d <= "20170427")     .where(appUpdateChannel="nightly")     .records(sc)


# In[94]:

data = get_pings_properties(pings, [
        "clientId",
        "environment/system/os/name",
        "environment/addons/activeExperiment",
        "environment/addons/activePlugins",
        "payload/info/subsessionLength",
        "payload/histograms/INPUT_EVENT_RESPONSE_MS",
        "payload/keyedHistograms/BLOCKED_ON_PLUGIN_INSTANCE_INIT_MS",
        "payload/keyedHistograms/BLOCKED_ON_PLUGIN_STREAM_INIT_MS",
        "payload/keyedHistograms/BLOCKED_ON_PLUGIN_INSTANCE_DESTROY_MS",
        "payload/histograms/PLUGINS_NOTIFICATION_SHOWN",
        "payload/histograms/PLUGINS_NOTIFICATION_USER_ACTION",
        "payload/histograms/PLUGINS_INFOBAR_SHOWN",
        "payload/histograms/PLUGINS_INFOBAR_BLOCK",
        "payload/histograms/PLUGINS_INFOBAR_ALLOW",
        "payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/plugin",
        "payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/content",
        "payload/keyedHistograms/SUBPROCESS_KILL_HARD/ShutDownKill",
    ], with_processes=True)


# In[95]:

def add_nullq(a, b):
    if a is None:
        return b
    if b is None:
        return a
    return a + b

def max_nullq(a, b):
    if a is None:
        return b
    if b is None:
        return a
    return max(a, b)

def import_keyedh(d):
    """
    The BLOCKED_ON_* histograms are keyed on the plugin name with version, which
    is pretty pathetic, and since we're only running Flash anyway it doesn't matter. So
    at import time we combine any/all of the values."""
    if d is None:
        return None
    return reduce(add_nullq, d.values())

props = (
    ('session_length', 'payload/info/subsessionLength', add_nullq, None),
    ('input_event_response_chrome', 'payload/histograms/INPUT_EVENT_RESPONSE_MS_parent', add_nullq, None),
    ('input_event_response_content', 'payload/histograms/INPUT_EVENT_RESPONSE_MS_children', add_nullq, None),
    ('blocked_on_instance_init', "payload/keyedHistograms/BLOCKED_ON_PLUGIN_INSTANCE_INIT_MS", add_nullq, import_keyedh),
    ('blocked_on_stream_init', "payload/keyedHistograms/BLOCKED_ON_PLUGIN_STREAM_INIT_MS", add_nullq, import_keyedh),
    ('blocked_on_instance_destroy', "payload/keyedHistograms/BLOCKED_ON_PLUGIN_INSTANCE_DESTROY_MS", add_nullq, import_keyedh),
    ('plugins_notification_shown', "payload/histograms/PLUGINS_NOTIFICATION_SHOWN", add_nullq, None),
    ('plugins_notification_user_action', "payload/histograms/PLUGINS_NOTIFICATION_USER_ACTION", add_nullq, None),
    ('plugins_infobar_shown', "payload/histograms/PLUGINS_INFOBAR_SHOWN", add_nullq, None),
    ('plugins_infobar_block', "payload/histograms/PLUGINS_INFOBAR_BLOCK", add_nullq, None),
    ('plugins_infobar_allow', "payload/histograms/PLUGINS_INFOBAR_ALLOW", add_nullq, None),
    ('plugin_crashes', "payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/plugin", add_nullq, None),
    ('content_crashes', "payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/content", add_nullq, None),
    ('shutdown_kill', "payload/keyedHistograms/SUBPROCESS_KILL_HARD/ShutDownKill", add_nullq, None),
)

PingData = namedtuple("PingData", (p for p, h, o, f in props))

def json_to_pingdata(d):
    return PingData._make((f or (lambda i: i))(d[h]) for p, h, o, f in props)

def add_pingdata(a, b):
    return PingData._make(o(va, vb) for (p, h, o, f), va, vb in zip(props, a, b))

def ping_has_flash(p):
    plugins = p["environment/addons/activePlugins"]
    if plugins is None:
        return False
    for plugin in plugins:
        if any(i == "application/x-shockwave-flash" for i in plugin.get("mimeTypes", [])):
            return True
    return False

def ping_filter(p):
    if p.get("payload/info/subsessionLength", 0) <= 0:
        return []
    e = p["environment/addons/activeExperiment"]
    if e is None:
        return []
    if e.get("id", None) != "flash-as-ctp-nightly55@experiments.mozilla.org":
        return []
    
    # The test group is split into "test" and "no-flash", but the control group
    # is not. Do that artificially here by inspecting activePlugins.
    branch = e.get("branch", None)
    if branch == "control" and not ping_has_flash(p):
        branch = "control-no-flash"
    return [(branch, p['clientId'], json_to_pingdata(p))]

data2 = data.flatMap(ping_filter).cache()


# In[96]:

data2.map(lambda (branch, id, d): branch).countByValue()


# In[97]:

bybranch = data2.map(lambda (branch, id, d): (branch, d)).reduceByKey(add_pingdata)


# In[98]:

branch_data = dict(bybranch.collect())


# In[101]:

print branch_data['control'].session_length
print branch_data['test'].session_length
print branch_data['test-waiting'].session_length


# In[102]:

# how many total plugin instances were there in each group?
print "control: ", branch_data['control'].blocked_on_instance_init.sum()
print "test: ", branch_data['test'].blocked_on_instance_init.sum()


# In[123]:

# what did this do to plugin init/stream/destroy jank?

def reduce_jank(cohort):
    d = branch_data[cohort]
    return reduce(add_nullq, [d.blocked_on_instance_init, d.blocked_on_stream_init, d.blocked_on_instance_destroy])
    
control_jank = reduce_jank('control')
test_jank = reduce_jank('test')
for (cutoff, control_count), (cutoff2, test_count) in zip(control_jank.iteritems(), test_jank.iteritems()):
    print "{:5d}ms: {:6.0f} {:6.0f}  ({:+.1f}%)".format(
        cutoff,
        control_count,
        test_count,
        (float(test_count) - control_count) / control_count * 100
    )


# In[103]:

# if we count opening with the plugin dialog as a "failure", how often does this happen?
print "{:.1f} hours".format(float(branch_data['test'].session_length) / 60 / 60 / branch_data['test'].plugins_notification_shown.sum())


# In[113]:

# How often do users take an action on the plugin dialog?
print "{:.0f}%".format(float(branch_data['test'].plugins_notification_user_action.sum()) / branch_data['test'].plugins_notification_shown.sum() * 100)


# In[106]:

# Which action do they take on the dialog?
actions = branch_data['test'].plugins_notification_user_action
print "User chose 'Allow Now': {:d}".format(int(actions[0]))
print "User chose 'Allow and Remember': {:d}".format(int(actions[1]))
print "User chose 'Block': {:d}".format(int(actions[2]))


# In[124]:

# if we count seeing the infobar as a failure, how often does this happen?
total_infobar = branch_data['test'].plugins_infobar_shown.sum()
print "{:.1f} hours".format(branch_data['test'].session_length / 60 / 60 / total_infobar)


# In[128]:

# What actions to users take on the infobar?
print "allow: {:.1f}%".format(branch_data['test'].plugins_infobar_allow.sum() / total_infobar * 100)
print "block: {:.1f}%".format(branch_data['test'].plugins_infobar_block.sum() / total_infobar * 100)
print "(no action): {:.1f}%".format((total_infobar - branch_data['test'].plugins_infobar_allow.sum() - branch_data['test'].plugins_infobar_block.sum()) / total_infobar * 100)


# In[107]:

print branch_data['control'].plugin_crashes
print branch_data['test'].plugin_crashes


# In[108]:

print branch_data['control'].content_crashes - branch_data['control'].shutdown_kill
print branch_data['test'].content_crashes - branch_data['test'].shutdown_kill


# In[109]:

print branch_data['control'].shutdown_kill
print branch_data['test'].shutdown_kill


# In[111]:

data2.saveAsPickleFile("s3n://net-mozaws-prod-us-west-2-pipeline-analysis/bsmedberg/2017-04-28-flash-experiment2")