flash-experiment-nightly-2017

flash-experiment-nightly-2017.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import ujson as json\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import plotly.plotly as py\n",
    "\n",
    "from plotly.graph_objs import *\n",
    "from moztelemetry import get_pings_properties\n",
    "from moztelemetry.dataset import Dataset\n",
    "from moztelemetry.histogram import Histogram\n",
    "from operator import add\n",
    "\n",
    "from datetime import date, timedelta\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import namedtuple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pings = Dataset.from_source(\"telemetry\") \\\n",
    "    .where(docType=\"main\") \\\n",
    "    .where(submissionDate=lambda d: \"20170413\" <= d <= \"20170427\") \\\n",
    "    .where(appUpdateChannel=\"nightly\") \\\n",
    "    .records(sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = get_pings_properties(pings, [\n",
    "        \"clientId\",\n",
    "        \"environment/system/os/name\",\n",
    "        \"environment/addons/activeExperiment\",\n",
    "        \"environment/addons/activePlugins\",\n",
    "        \"payload/info/subsessionLength\",\n",
    "        \"payload/histograms/INPUT_EVENT_RESPONSE_MS\",\n",
    "        \"payload/keyedHistograms/BLOCKED_ON_PLUGIN_INSTANCE_INIT_MS\",\n",
    "        \"payload/keyedHistograms/BLOCKED_ON_PLUGIN_STREAM_INIT_MS\",\n",
    "        \"payload/keyedHistograms/BLOCKED_ON_PLUGIN_INSTANCE_DESTROY_MS\",\n",
    "        \"payload/histograms/PLUGINS_NOTIFICATION_SHOWN\",\n",
    "        \"payload/histograms/PLUGINS_NOTIFICATION_USER_ACTION\",\n",
    "        \"payload/histograms/PLUGINS_INFOBAR_SHOWN\",\n",
    "        \"payload/histograms/PLUGINS_INFOBAR_BLOCK\",\n",
    "        \"payload/histograms/PLUGINS_INFOBAR_ALLOW\",\n",
    "        \"payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/plugin\",\n",
    "        \"payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/content\",\n",
    "        \"payload/keyedHistograms/SUBPROCESS_KILL_HARD/ShutDownKill\",\n",
    "    ], with_processes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def add_nullq(a, b):\n",
    "    if a is None:\n",
    "        return b\n",
    "    if b is None:\n",
    "        return a\n",
    "    return a + b\n",
    "\n",
    "def max_nullq(a, b):\n",
    "    if a is None:\n",
    "        return b\n",
    "    if b is None:\n",
    "        return a\n",
    "    return max(a, b)\n",
    "\n",
    "def import_keyedh(d):\n",
    "    \"\"\"\n",
    "    The BLOCKED_ON_* histograms are keyed on the plugin name with version, which\n",
    "    is pretty pathetic, and since we're only running Flash anyway it doesn't matter. So\n",
    "    at import time we combine any/all of the values.\"\"\"\n",
    "    if d is None:\n",
    "        return None\n",
    "    return reduce(add_nullq, d.values())\n",
    "\n",
    "props = (\n",
    "    ('session_length', 'payload/info/subsessionLength', add_nullq, None),\n",
    "    ('input_event_response_chrome', 'payload/histograms/INPUT_EVENT_RESPONSE_MS_parent', add_nullq, None),\n",
    "    ('input_event_response_content', 'payload/histograms/INPUT_EVENT_RESPONSE_MS_children', add_nullq, None),\n",
    "    ('blocked_on_instance_init', \"payload/keyedHistograms/BLOCKED_ON_PLUGIN_INSTANCE_INIT_MS\", add_nullq, import_keyedh),\n",
    "    ('blocked_on_stream_init', \"payload/keyedHistograms/BLOCKED_ON_PLUGIN_STREAM_INIT_MS\", add_nullq, import_keyedh),\n",
    "    ('blocked_on_instance_destroy', \"payload/keyedHistograms/BLOCKED_ON_PLUGIN_INSTANCE_DESTROY_MS\", add_nullq, import_keyedh),\n",
    "    ('plugins_notification_shown', \"payload/histograms/PLUGINS_NOTIFICATION_SHOWN\", add_nullq, None),\n",
    "    ('plugins_notification_user_action', \"payload/histograms/PLUGINS_NOTIFICATION_USER_ACTION\", add_nullq, None),\n",
    "    ('plugins_infobar_shown', \"payload/histograms/PLUGINS_INFOBAR_SHOWN\", add_nullq, None),\n",
    "    ('plugins_infobar_block', \"payload/histograms/PLUGINS_INFOBAR_BLOCK\", add_nullq, None),\n",
    "    ('plugins_infobar_allow', \"payload/histograms/PLUGINS_INFOBAR_ALLOW\", add_nullq, None),\n",
    "    ('plugin_crashes', \"payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/plugin\", add_nullq, None),\n",
    "    ('content_crashes', \"payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/content\", add_nullq, None),\n",
    "    ('shutdown_kill', \"payload/keyedHistograms/SUBPROCESS_KILL_HARD/ShutDownKill\", add_nullq, None),\n",
    ")\n",
    "\n",
    "PingData = namedtuple(\"PingData\", (p for p, h, o, f in props))\n",
    "\n",
    "def json_to_pingdata(d):\n",
    "    return PingData._make((f or (lambda i: i))(d[h]) for p, h, o, f in props)\n",
    "\n",
    "def add_pingdata(a, b):\n",
    "    return PingData._make(o(va, vb) for (p, h, o, f), va, vb in zip(props, a, b))\n",
    "\n",
    "def ping_has_flash(p):\n",
    "    plugins = p[\"environment/addons/activePlugins\"]\n",
    "    if plugins is None:\n",
    "        return False\n",
    "    for plugin in plugins:\n",
    "        if any(i == \"application/x-shockwave-flash\" for i in plugin.get(\"mimeTypes\", [])):\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "def ping_filter(p):\n",
    "    if p.get(\"payload/info/subsessionLength\", 0) <= 0:\n",
    "        return []\n",
    "    e = p[\"environment/addons/activeExperiment\"]\n",
    "    if e is None:\n",
    "        return []\n",
    "    if e.get(\"id\", None) != \"flash-as-ctp-nightly55@experiments.mozilla.org\":\n",
    "        return []\n",
    "    \n",
    "    # The test group is split into \"test\" and \"no-flash\", but the control group\n",
    "    # is not. Do that artificially here by inspecting activePlugins.\n",
    "    branch = e.get(\"branch\", None)\n",
    "    if branch == \"control\" and not ping_has_flash(p):\n",
    "        branch = \"control-no-flash\"\n",
    "    return [(branch, p['clientId'], json_to_pingdata(p))]\n",
    "\n",
    "data2 = data.flatMap(ping_filter).cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(int,\n",
       "            {u'control': 55144,\n",
       "             'control-no-flash': 22372,\n",
       "             u'disqualified': 12953,\n",
       "             u'disqualified-prefchanged': 1244,\n",
       "             u'no-flash': 19418,\n",
       "             u'test': 46862,\n",
       "             u'test-waiting': 10226})"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2.map(lambda (branch, id, d): branch).countByValue()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bybranch = data2.map(lambda (branch, id, d): (branch, d)).reduceByKey(add_pingdata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "branch_data = dict(bybranch.collect())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "511862289\n",
      "429072580\n",
      "113667478\n"
     ]
    }
   ],
   "source": [
    "print branch_data['control'].session_length\n",
    "print branch_data['test'].session_length\n",
    "print branch_data['test-waiting'].session_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "control:  31476.0\n",
      "test:  7482.0\n"
     ]
    }
   ],
   "source": [
    "# how many total plugin instances were there in each group?\n",
    "print \"control: \", branch_data['control'].blocked_on_instance_init.sum()\n",
    "print \"test: \", branch_data['test'].blocked_on_instance_init.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    0ms: 166048   8854  (-94.7%)\n",
      "    1ms:  85198  59438  (-30.2%)\n",
      "    2ms:  20350  19742  (-3.0%)\n",
      "    3ms:  16614   8730  (-47.5%)\n",
      "    5ms:  15704   3004  (-80.9%)\n",
      "    8ms:  17376   1920  (-89.0%)\n",
      "   13ms:  24110   5382  (-77.7%)\n",
      "   22ms:  29536   3480  (-88.2%)\n",
      "   37ms:  22406   3648  (-83.7%)\n",
      "   62ms:  17602   3506  (-80.1%)\n",
      "  103ms:  10762   4668  (-56.6%)\n",
      "  171ms:   4700    992  (-78.9%)\n",
      "  284ms:   2742    644  (-76.5%)\n",
      "  472ms:   1002    454  (-54.7%)\n",
      "  785ms:    626    172  (-72.5%)\n",
      " 1306ms:    260    124  (-52.3%)\n",
      " 2172ms:    208     66  (-68.3%)\n",
      " 3613ms:    108     26  (-75.9%)\n",
      " 6011ms:     36     10  (-72.2%)\n",
      "10000ms:     60      8  (-86.7%)\n"
     ]
    }
   ],
   "source": [
    "# what did this do to plugin init/stream/destroy jank?\n",
    "\n",
    "def reduce_jank(cohort):\n",
    "    d = branch_data[cohort]\n",
    "    return reduce(add_nullq, [d.blocked_on_instance_init, d.blocked_on_stream_init, d.blocked_on_instance_destroy])\n",
    "    \n",
    "control_jank = reduce_jank('control')\n",
    "test_jank = reduce_jank('test')\n",
    "for (cutoff, control_count), (cutoff2, test_count) in zip(control_jank.iteritems(), test_jank.iteritems()):\n",
    "    print \"{:5d}ms: {:6.0f} {:6.0f}  ({:+.1f}%)\".format(\n",
    "        cutoff,\n",
    "        control_count,\n",
    "        test_count,\n",
    "        (float(test_count) - control_count) / control_count * 100\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21.8 hours\n"
     ]
    }
   ],
   "source": [
    "# if we count opening with the plugin dialog as a \"failure\", how often does this happen?\n",
    "print \"{:.1f} hours\".format(float(branch_data['test'].session_length) / 60 / 60 / branch_data['test'].plugins_notification_shown.sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "83%\n"
     ]
    }
   ],
   "source": [
    "# How often do users take an action on the plugin dialog?\n",
    "print \"{:.0f}%\".format(float(branch_data['test'].plugins_notification_user_action.sum()) / branch_data['test'].plugins_notification_shown.sum() * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "User chose 'Allow Now': 2700\n",
      "User chose 'Allow and Remember': 1858\n",
      "User chose 'Block': 10\n"
     ]
    }
   ],
   "source": [
    "# Which action do they take on the dialog?\n",
    "actions = branch_data['test'].plugins_notification_user_action\n",
    "print \"User chose 'Allow Now': {:d}\".format(int(actions[0]))\n",
    "print \"User chose 'Allow and Remember': {:d}\".format(int(actions[1]))\n",
    "print \"User chose 'Block': {:d}\".format(int(actions[2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.5 hours\n"
     ]
    }
   ],
   "source": [
    "# if we count seeing the infobar as a failure, how often does this happen?\n",
    "total_infobar = branch_data['test'].plugins_infobar_shown.sum()\n",
    "print \"{:.1f} hours\".format(branch_data['test'].session_length / 60 / 60 / total_infobar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "allow: 7.1%\n",
      "block: 0.6%\n",
      "(no action): 92.3%\n"
     ]
    }
   ],
   "source": [
    "# What actions to users take on the infobar?\n",
    "print \"allow: {:.1f}%\".format(branch_data['test'].plugins_infobar_allow.sum() / total_infobar * 100)\n",
    "print \"block: {:.1f}%\".format(branch_data['test'].plugins_infobar_block.sum() / total_infobar * 100)\n",
    "print \"(no action): {:.1f}%\".format((total_infobar - branch_data['test'].plugins_infobar_allow.sum() - branch_data['test'].plugins_infobar_block.sum()) / total_infobar * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "143\n",
      "96\n"
     ]
    }
   ],
   "source": [
    "print branch_data['control'].plugin_crashes\n",
    "print branch_data['test'].plugin_crashes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "236\n",
      "271\n"
     ]
    }
   ],
   "source": [
    "print branch_data['control'].content_crashes - branch_data['control'].shutdown_kill\n",
    "print branch_data['test'].content_crashes - branch_data['test'].shutdown_kill"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2334\n",
      "1917\n"
     ]
    }
   ],
   "source": [
    "print branch_data['control'].shutdown_kill\n",
    "print branch_data['test'].shutdown_kill"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data2.saveAsPickleFile(\"s3n://net-mozaws-prod-us-west-2-pipeline-analysis/bsmedberg/2017-04-28-flash-experiment2\")"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}

flash-experiment-nightly-2017.py

# coding: utf-8

# In[17]:

import ujson as json
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import plotly.plotly as py

from plotly.graph_objs import *
from moztelemetry import get_pings_properties
from moztelemetry.dataset import Dataset
from moztelemetry.histogram import Histogram
from operator import add

from datetime import date, timedelta

get_ipython().magic(u'matplotlib inline')


# In[18]:

from collections import namedtuple


# In[19]:

pings = Dataset.from_source("telemetry")     .where(docType="main")     .where(submissionDate=lambda d: "20170413" <= d <= "20170427")     .where(appUpdateChannel="nightly")     .records(sc)


# In[94]:

data = get_pings_properties(pings, [
        "clientId",
        "environment/system/os/name",
        "environment/addons/activeExperiment",
        "environment/addons/activePlugins",
        "payload/info/subsessionLength",
        "payload/histograms/INPUT_EVENT_RESPONSE_MS",
        "payload/keyedHistograms/BLOCKED_ON_PLUGIN_INSTANCE_INIT_MS",
        "payload/keyedHistograms/BLOCKED_ON_PLUGIN_STREAM_INIT_MS",
        "payload/keyedHistograms/BLOCKED_ON_PLUGIN_INSTANCE_DESTROY_MS",
        "payload/histograms/PLUGINS_NOTIFICATION_SHOWN",
        "payload/histograms/PLUGINS_NOTIFICATION_USER_ACTION",
        "payload/histograms/PLUGINS_INFOBAR_SHOWN",
        "payload/histograms/PLUGINS_INFOBAR_BLOCK",
        "payload/histograms/PLUGINS_INFOBAR_ALLOW",
        "payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/plugin",
        "payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/content",
        "payload/keyedHistograms/SUBPROCESS_KILL_HARD/ShutDownKill",
    ], with_processes=True)


# In[95]:

def add_nullq(a, b):
    if a is None:
        return b
    if b is None:
        return a
    return a + b

def max_nullq(a, b):
    if a is None:
        return b
    if b is None:
        return a
    return max(a, b)

def import_keyedh(d):
    """
    The BLOCKED_ON_* histograms are keyed on the plugin name with version, which
    is pretty pathetic, and since we're only running Flash anyway it doesn't matter. So
    at import time we combine any/all of the values."""
    if d is None:
        return None
    return reduce(add_nullq, d.values())

props = (
    ('session_length', 'payload/info/subsessionLength', add_nullq, None),
    ('input_event_response_chrome', 'payload/histograms/INPUT_EVENT_RESPONSE_MS_parent', add_nullq, None),
    ('input_event_response_content', 'payload/histograms/INPUT_EVENT_RESPONSE_MS_children', add_nullq, None),
    ('blocked_on_instance_init', "payload/keyedHistograms/BLOCKED_ON_PLUGIN_INSTANCE_INIT_MS", add_nullq, import_keyedh),
    ('blocked_on_stream_init', "payload/keyedHistograms/BLOCKED_ON_PLUGIN_STREAM_INIT_MS", add_nullq, import_keyedh),
    ('blocked_on_instance_destroy', "payload/keyedHistograms/BLOCKED_ON_PLUGIN_INSTANCE_DESTROY_MS", add_nullq, import_keyedh),
    ('plugins_notification_shown', "payload/histograms/PLUGINS_NOTIFICATION_SHOWN", add_nullq, None),
    ('plugins_notification_user_action', "payload/histograms/PLUGINS_NOTIFICATION_USER_ACTION", add_nullq, None),
    ('plugins_infobar_shown', "payload/histograms/PLUGINS_INFOBAR_SHOWN", add_nullq, None),
    ('plugins_infobar_block', "payload/histograms/PLUGINS_INFOBAR_BLOCK", add_nullq, None),
    ('plugins_infobar_allow', "payload/histograms/PLUGINS_INFOBAR_ALLOW", add_nullq, None),
    ('plugin_crashes', "payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/plugin", add_nullq, None),
    ('content_crashes', "payload/keyedHistograms/SUBPROCESS_CRASHES_WITH_DUMP/content", add_nullq, None),
    ('shutdown_kill', "payload/keyedHistograms/SUBPROCESS_KILL_HARD/ShutDownKill", add_nullq, None),
)

PingData = namedtuple("PingData", (p for p, h, o, f in props))

def json_to_pingdata(d):
    return PingData._make((f or (lambda i: i))(d[h]) for p, h, o, f in props)

def add_pingdata(a, b):
    return PingData._make(o(va, vb) for (p, h, o, f), va, vb in zip(props, a, b))

def ping_has_flash(p):
    plugins = p["environment/addons/activePlugins"]
    if plugins is None:
        return False
    for plugin in plugins:
        if any(i == "application/x-shockwave-flash" for i in plugin.get("mimeTypes", [])):
            return True
    return False

def ping_filter(p):
    if p.get("payload/info/subsessionLength", 0) <= 0:
        return []
    e = p["environment/addons/activeExperiment"]
    if e is None:
        return []
    if e.get("id", None) != "flash-as-ctp-nightly55@experiments.mozilla.org":
        return []
    
    # The test group is split into "test" and "no-flash", but the control group
    # is not. Do that artificially here by inspecting activePlugins.
    branch = e.get("branch", None)
    if branch == "control" and not ping_has_flash(p):
        branch = "control-no-flash"
    return [(branch, p['clientId'], json_to_pingdata(p))]

data2 = data.flatMap(ping_filter).cache()


# In[96]:

data2.map(lambda (branch, id, d): branch).countByValue()


# In[97]:

bybranch = data2.map(lambda (branch, id, d): (branch, d)).reduceByKey(add_pingdata)


# In[98]:

branch_data = dict(bybranch.collect())


# In[101]:

print branch_data['control'].session_length
print branch_data['test'].session_length
print branch_data['test-waiting'].session_length


# In[102]:

# how many total plugin instances were there in each group?
print "control: ", branch_data['control'].blocked_on_instance_init.sum()
print "test: ", branch_data['test'].blocked_on_instance_init.sum()


# In[123]:

# what did this do to plugin init/stream/destroy jank?

def reduce_jank(cohort):
    d = branch_data[cohort]
    return reduce(add_nullq, [d.blocked_on_instance_init, d.blocked_on_stream_init, d.blocked_on_instance_destroy])
    
control_jank = reduce_jank('control')
test_jank = reduce_jank('test')
for (cutoff, control_count), (cutoff2, test_count) in zip(control_jank.iteritems(), test_jank.iteritems()):
    print "{:5d}ms: {:6.0f} {:6.0f}  ({:+.1f}%)".format(
        cutoff,
        control_count,
        test_count,
        (float(test_count) - control_count) / control_count * 100
    )


# In[103]:

# if we count opening with the plugin dialog as a "failure", how often does this happen?
print "{:.1f} hours".format(float(branch_data['test'].session_length) / 60 / 60 / branch_data['test'].plugins_notification_shown.sum())


# In[113]:

# How often do users take an action on the plugin dialog?
print "{:.0f}%".format(float(branch_data['test'].plugins_notification_user_action.sum()) / branch_data['test'].plugins_notification_shown.sum() * 100)


# In[106]:

# Which action do they take on the dialog?
actions = branch_data['test'].plugins_notification_user_action
print "User chose 'Allow Now': {:d}".format(int(actions[0]))
print "User chose 'Allow and Remember': {:d}".format(int(actions[1]))
print "User chose 'Block': {:d}".format(int(actions[2]))


# In[124]:

# if we count seeing the infobar as a failure, how often does this happen?
total_infobar = branch_data['test'].plugins_infobar_shown.sum()
print "{:.1f} hours".format(branch_data['test'].session_length / 60 / 60 / total_infobar)


# In[128]:

# What actions to users take on the infobar?
print "allow: {:.1f}%".format(branch_data['test'].plugins_infobar_allow.sum() / total_infobar * 100)
print "block: {:.1f}%".format(branch_data['test'].plugins_infobar_block.sum() / total_infobar * 100)
print "(no action): {:.1f}%".format((total_infobar - branch_data['test'].plugins_infobar_allow.sum() - branch_data['test'].plugins_infobar_block.sum()) / total_infobar * 100)


# In[107]:

print branch_data['control'].plugin_crashes
print branch_data['test'].plugin_crashes


# In[108]:

print branch_data['control'].content_crashes - branch_data['control'].shutdown_kill
print branch_data['test'].content_crashes - branch_data['test'].shutdown_kill


# In[109]:

print branch_data['control'].shutdown_kill
print branch_data['test'].shutdown_kill


# In[111]:

data2.saveAsPickleFile("s3n://net-mozaws-prod-us-west-2-pipeline-analysis/bsmedberg/2017-04-28-flash-experiment2")