Beta 45 Experiment: crash rates using SUBPROCESS_CRASHES_WITH_DUMP from over the first weekend of the experiment

beta45_crash_rates_with_dump.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### E10S Experiment Beta 45 (with addons): Crash rate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Bug 1222890](https://bugzilla.mozilla.org/show_bug.cgi?id=1222890)\n",
    "\n",
    "This analysis compares e10s and non-e10s crash rates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unable to parse whitelist (/home/hadoop/anaconda2/lib/python2.7/site-packages/moztelemetry/bucket-whitelist.json). Assuming all histograms are acceptable.\n",
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "import ujson as json\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import plotly.plotly as py\n",
    "import IPython\n",
    "\n",
    "from __future__ import division\n",
    "from moztelemetry.spark import get_pings, get_one_ping_per_client, get_pings_properties\n",
    "from montecarlino import grouped_permutation_test\n",
    "\n",
    "%pylab inline\n",
    "IPython.core.pylabtools.figsize(16, 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc.defaultParallelism"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def is_in_e10s_experiment(ping):\n",
    "    try:\n",
    "        experiment = ping[\"environment\"][\"addons\"][\"activeExperiment\"]\n",
    "        return experiment[\"id\"] == \"e10s-beta45-withaddons@experiments.mozilla.org\" and \\\n",
    "               (experiment[\"branch\"] == \"control\" or experiment[\"branch\"] == \"experiment\")   \n",
    "    except:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def is_e10s_ping(ping):\n",
    "    return ping[\"environment\"][\"settings\"][\"e10sEnabled\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "PING_OPTIONS = { \"app\": \"Firefox\", \"channel\": \"beta\", \"version\": \"45.0\", \"submission_date\": (\"20160125\", \"20160201\") }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "main_pings = get_pings(sc, doc_type=\"main\", **PING_OPTIONS).filter(is_in_e10s_experiment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "crash_pings = get_pings(sc, doc_type=\"crash\", **PING_OPTIONS).filter(is_in_e10s_experiment)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What are the total subsession lengths per build ID?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_subsession_lengths_per_build_id(pings):\n",
    "    return pings.map(lambda p: (p[\"application\"][\"buildId\"], p[\"payload\"][\"info\"][\"subsessionLength\"])) \\\n",
    "                .reduceByKey(lambda a, b: a + b).collectAsMap()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{u'20160124191701': 9031,\n",
       " u'20160124191723': 4818,\n",
       " u'20160127070712': 2891605067,\n",
       " u'20160128103813': 1895}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "e10s_main_pings = main_pings.filter(lambda p: is_e10s_ping(p))\n",
    "e10s_subsession_lengths = get_subsession_lengths_per_build_id(e10s_main_pings)\n",
    "e10s_subsession_lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{u'20160120200200': 169214,\n",
       " u'20160124191701': 4779,\n",
       " u'20160124191723': 555,\n",
       " u'20160124191801': 3596,\n",
       " u'20160127070712': 3179248699,\n",
       " u'20160128104122': 11525}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "non_e10s_main_pings = main_pings.filter(lambda p: not is_e10s_ping(p))\n",
    "non_e10s_subsession_lengths = get_subsession_lengths_per_build_id(non_e10s_main_pings)\n",
    "non_e10s_subsession_lengths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What are the total (parent) crash counts per build ID?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_crash_counts_per_build_id(pings):\n",
    "    return dict(pings.map(lambda p: (p[\"application\"][\"buildId\"], 0)).countByKey())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{u'20160124191701': 2,\n",
       " u'20160124191723': 2,\n",
       " u'20160124191801': 1,\n",
       " u'20160127070712': 11638}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "e10s_crash_pings = crash_pings.filter(lambda p: is_e10s_ping(p))\n",
    "e10s_crash_counts = get_crash_counts_per_build_id(e10s_crash_pings)\n",
    "e10s_crash_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{u'20160127070712': 23312}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "non_e10s_crash_pings = crash_pings.filter(lambda p: not is_e10s_ping(p))\n",
    "non_e10s_crash_counts = get_crash_counts_per_build_id(non_e10s_crash_pings)\n",
    "non_e10s_crash_counts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What are the total content crash counts per build ID?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_content_abort_count(ping):\n",
    "    return ping[\"payload\"].get(\"keyedHistograms\", {}).get(\"SUBPROCESS_CRASHES_WITH_DUMP\", {}).get(\"content\", {}).get(\"sum\", 0)\n",
    "\n",
    "def get_content_crash_count_per_build_id(pings):\n",
    "    return pings.map(lambda p: (p[\"application\"][\"buildId\"], get_content_abort_count(p))) \\\n",
    "                .reduceByKey(lambda a, b: a + b).collectAsMap()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{u'20160124191701': 2,\n",
       " u'20160124191723': 2,\n",
       " u'20160127070712': 30250,\n",
       " u'20160128103813': 0}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "e10s_content_crash_counts = get_content_crash_count_per_build_id(e10s_main_pings)\n",
    "e10s_content_crash_counts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Crashes per 1000 usage hours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "build ID             non-e10s    e10s-parent   e10s-content\n",
      "20160127070712         26.397         14.489         37.661\n"
     ]
    }
   ],
   "source": [
    "SECS_PER_1000_HOURS = 1000 * 60 * 60\n",
    "print \"build ID             non-e10s    e10s-parent   e10s-content\"\n",
    "for build_id in sorted(set(e10s_crash_counts.keys()) & set(non_e10s_crash_counts.keys())):\n",
    "    print \"{} {:>14.3f} {:>14.3f} {:>14.3f}\".format(\n",
    "        build_id,\n",
    "        non_e10s_crash_counts[build_id] / non_e10s_subsession_lengths[build_id] * SECS_PER_1000_HOURS,\n",
    "        e10s_crash_counts[build_id] / e10s_subsession_lengths[build_id] * SECS_PER_1000_HOURS,\n",
    "        e10s_content_crash_counts[build_id] / e10s_subsession_lengths[build_id] * SECS_PER_1000_HOURS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}