Created
February 1, 2016 19:32
-
-
Save chutten/e80d7f2f1a52f07e642b to your computer and use it in GitHub Desktop.
Beta 45 Experiment: crash rates using SUBPROCESS_CRASHES_WITH_DUMP from over the first weekend of the experiment
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"### E10S Experiment Beta 45 (with addons): Crash rate" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"[Bug 1222890](https://bugzilla.mozilla.org/show_bug.cgi?id=1222890)\n", | |
"\n", | |
"This analysis compares e10s and non-e10s crash rates." | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 1, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"Unable to parse whitelist (/home/hadoop/anaconda2/lib/python2.7/site-packages/moztelemetry/bucket-whitelist.json). Assuming all histograms are acceptable.\n", | |
"Populating the interactive namespace from numpy and matplotlib\n" | |
] | |
} | |
], | |
"source": [ | |
"import ujson as json\n", | |
"import matplotlib.pyplot as plt\n", | |
"import pandas as pd\n", | |
"import numpy as np\n", | |
"import plotly.plotly as py\n", | |
"import IPython\n", | |
"\n", | |
"from __future__ import division\n", | |
"from moztelemetry.spark import get_pings, get_one_ping_per_client, get_pings_properties\n", | |
"from montecarlino import grouped_permutation_test\n", | |
"\n", | |
"%pylab inline\n", | |
"IPython.core.pylabtools.figsize(16, 7)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 2, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"32" | |
] | |
}, | |
"execution_count": 2, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"sc.defaultParallelism" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 3, | |
"metadata": { | |
"collapsed": true | |
}, | |
"outputs": [], | |
"source": [ | |
"def is_in_e10s_experiment(ping):\n", | |
" try:\n", | |
" experiment = ping[\"environment\"][\"addons\"][\"activeExperiment\"]\n", | |
" return experiment[\"id\"] == \"e10s-beta45-withaddons@experiments.mozilla.org\" and \\\n", | |
" (experiment[\"branch\"] == \"control\" or experiment[\"branch\"] == \"experiment\") \n", | |
" except:\n", | |
" return False" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 4, | |
"metadata": { | |
"collapsed": true | |
}, | |
"outputs": [], | |
"source": [ | |
"def is_e10s_ping(ping):\n", | |
" return ping[\"environment\"][\"settings\"][\"e10sEnabled\"]" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"### Pings" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 5, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [], | |
"source": [ | |
"PING_OPTIONS = { \"app\": \"Firefox\", \"channel\": \"beta\", \"version\": \"45.0\", \"submission_date\": (\"20160125\", \"20160201\") }" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 6, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [], | |
"source": [ | |
"main_pings = get_pings(sc, doc_type=\"main\", **PING_OPTIONS).filter(is_in_e10s_experiment)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 7, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [], | |
"source": [ | |
"crash_pings = get_pings(sc, doc_type=\"crash\", **PING_OPTIONS).filter(is_in_e10s_experiment)" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"What are the total subsession lengths per build ID?" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 8, | |
"metadata": { | |
"collapsed": true | |
}, | |
"outputs": [], | |
"source": [ | |
"def get_subsession_lengths_per_build_id(pings):\n", | |
" return pings.map(lambda p: (p[\"application\"][\"buildId\"], p[\"payload\"][\"info\"][\"subsessionLength\"])) \\\n", | |
" .reduceByKey(lambda a, b: a + b).collectAsMap()" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 9, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"{u'20160124191701': 9031,\n", | |
" u'20160124191723': 4818,\n", | |
" u'20160127070712': 2891605067,\n", | |
" u'20160128103813': 1895}" | |
] | |
}, | |
"execution_count": 9, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"e10s_main_pings = main_pings.filter(lambda p: is_e10s_ping(p))\n", | |
"e10s_subsession_lengths = get_subsession_lengths_per_build_id(e10s_main_pings)\n", | |
"e10s_subsession_lengths" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 10, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"{u'20160120200200': 169214,\n", | |
" u'20160124191701': 4779,\n", | |
" u'20160124191723': 555,\n", | |
" u'20160124191801': 3596,\n", | |
" u'20160127070712': 3179248699,\n", | |
" u'20160128104122': 11525}" | |
] | |
}, | |
"execution_count": 10, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"non_e10s_main_pings = main_pings.filter(lambda p: not is_e10s_ping(p))\n", | |
"non_e10s_subsession_lengths = get_subsession_lengths_per_build_id(non_e10s_main_pings)\n", | |
"non_e10s_subsession_lengths" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"What are the total (parent) crash counts per build ID?" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 11, | |
"metadata": { | |
"collapsed": true | |
}, | |
"outputs": [], | |
"source": [ | |
"def get_crash_counts_per_build_id(pings):\n", | |
" return dict(pings.map(lambda p: (p[\"application\"][\"buildId\"], 0)).countByKey())" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 12, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"{u'20160124191701': 2,\n", | |
" u'20160124191723': 2,\n", | |
" u'20160124191801': 1,\n", | |
" u'20160127070712': 11638}" | |
] | |
}, | |
"execution_count": 12, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"e10s_crash_pings = crash_pings.filter(lambda p: is_e10s_ping(p))\n", | |
"e10s_crash_counts = get_crash_counts_per_build_id(e10s_crash_pings)\n", | |
"e10s_crash_counts" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 13, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"{u'20160127070712': 23312}" | |
] | |
}, | |
"execution_count": 13, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"non_e10s_crash_pings = crash_pings.filter(lambda p: not is_e10s_ping(p))\n", | |
"non_e10s_crash_counts = get_crash_counts_per_build_id(non_e10s_crash_pings)\n", | |
"non_e10s_crash_counts" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"What are the total content crash counts per build ID?" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 14, | |
"metadata": { | |
"collapsed": true | |
}, | |
"outputs": [], | |
"source": [ | |
"def get_content_abort_count(ping):\n", | |
" return ping[\"payload\"].get(\"keyedHistograms\", {}).get(\"SUBPROCESS_CRASHES_WITH_DUMP\", {}).get(\"content\", {}).get(\"sum\", 0)\n", | |
"\n", | |
"def get_content_crash_count_per_build_id(pings):\n", | |
" return pings.map(lambda p: (p[\"application\"][\"buildId\"], get_content_abort_count(p))) \\\n", | |
" .reduceByKey(lambda a, b: a + b).collectAsMap()" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 15, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"{u'20160124191701': 2,\n", | |
" u'20160124191723': 2,\n", | |
" u'20160127070712': 30250,\n", | |
" u'20160128103813': 0}" | |
] | |
}, | |
"execution_count": 15, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"e10s_content_crash_counts = get_content_crash_count_per_build_id(e10s_main_pings)\n", | |
"e10s_content_crash_counts" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"### Crashes per 1000 usage hours" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 16, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"build ID non-e10s e10s-parent e10s-content\n", | |
"20160127070712 26.397 14.489 37.661\n" | |
] | |
} | |
], | |
"source": [ | |
"SECS_PER_1000_HOURS = 1000 * 60 * 60\n", | |
"print \"build ID non-e10s e10s-parent e10s-content\"\n", | |
"for build_id in sorted(set(e10s_crash_counts.keys()) & set(non_e10s_crash_counts.keys())):\n", | |
" print \"{} {:>14.3f} {:>14.3f} {:>14.3f}\".format(\n", | |
" build_id,\n", | |
" non_e10s_crash_counts[build_id] / non_e10s_subsession_lengths[build_id] * SECS_PER_1000_HOURS,\n", | |
" e10s_crash_counts[build_id] / e10s_subsession_lengths[build_id] * SECS_PER_1000_HOURS,\n", | |
" e10s_content_crash_counts[build_id] / e10s_subsession_lengths[build_id] * SECS_PER_1000_HOURS)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"metadata": { | |
"collapsed": true | |
}, | |
"outputs": [], | |
"source": [] | |
} | |
], | |
"metadata": { | |
"kernelspec": { | |
"display_name": "Python 2", | |
"language": "python", | |
"name": "python2" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 2 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython2", | |
"version": "2.7.10" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 0 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment