Skip to content

Instantly share code, notes, and snippets.

@chutten

chutten/first stacks.ipynb Secret

Last active January 26, 2017 19:26
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save chutten/2063fe236a6ed46eb9b566dfa51ea755 to your computer and use it in GitHub Desktop.
Save chutten/2063fe236a6ed46eb9b566dfa51ea755 to your computer and use it in GitHub Desktop.
Download ZIP
first stacks
Display the source blob
Display the rendered blob
Raw
first stacks.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### When a user crashes for the first time, what is that crash?"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": false,
"scrolled": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/hadoop/anaconda2/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
" warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Unable to parse whitelist (/home/hadoop/anaconda2/lib/python2.7/site-packages/moztelemetry/histogram-whitelists.json). Assuming all histograms are acceptable.\n"
]
}
],
"source": [
"import ujson as json\n",
"import matplotlib.pyplot as plt\n",
"import pandas as pd\n",
"import numpy as np\n",
"import plotly.plotly as py\n",
"import requests\n",
"\n",
"from plotly.graph_objs import *\n",
"from moztelemetry import get_pings_properties, get_one_ping_per_client\n",
"from moztelemetry.dataset import Dataset\n",
"\n",
"%matplotlib inline"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"320"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sc.defaultParallelism"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"pings = Dataset.from_source(\"telemetry\") \\\n",
" .where(docType='crash') \\\n",
" .where(appUpdateChannel=lambda c: c in ('nightly'))\\\n",
" .where(submissionDate=lambda d: d >= \"20170101\" and d <= \"20170112\") \\\n",
" .records(sc, sample=1)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"subset = get_pings_properties(pings, [\n",
" \"clientId\",\n",
" \"submissionDate\",\n",
" \"payload/stackTraces\",\n",
" \"environment/profile/creationDate\"])\\\n",
" .filter(lambda p: p[\"payload/stackTraces\"] is not None)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"8808"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"subset.count()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"First we need to filter out any profile older than our survey period."
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"import datetime\n",
"\n",
"survey_begin = (datetime.datetime(2017, 1, 1) - datetime.datetime(1970, 1, 1)).days\n",
"subset = subset.filter(lambda p: p[\"environment/profile/creationDate\"] >= survey_begin)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"508"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"subset.count()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"So now we have a survey subset of pings with stack information. We want to get the first crash for each client reporting a crash, so..."
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"firsts = subset\\\n",
" .map(lambda p: (p[\"clientId\"], p))\\\n",
" .reduceByKey(lambda a, b: a if a[\"submissionDate\"] < b[\"submissionDate\"] else b)\\\n",
" .map(lambda pair: pair[1])"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"216"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"firsts.count()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def symbolicate(s):\n",
" data = json.dumps({\n",
" 'stacks': [[[f['module_index'], int(f['ip'], 16) - int(s['modules'][f['module_index']]['base_addr'], 16)] for f in s['threads'][s['crash_info']['crashing_thread']]['frames']]],\n",
" 'memoryMap': [[m['debug_file'].translate(dict((ord(char), None) for char in ' ()')), m['debug_id']] for m in s['modules']], 'version': 4})\n",
" result = requests.post('http://symbolapi.mozilla.org/', data=data)\n",
" result_json = result.json()\n",
" return result_json['symbolicatedStacks']"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def safe_symbolicate(s):\n",
" try:\n",
" return symbolicate(s)\n",
" except Exception, e:\n",
" return e"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"symbolicated_firsts = firsts.map(lambda p: (p[\"environment/profile/creationDate\"], safe_symbolicate(p[\"payload/stackTraces\"])))"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"collapsed": false,
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"[(17167, KeyError('module_index')),\n",
" (17172,\n",
" [[u\"mozilla::`anonymous namespace'::RunWatchdog (in xul.pdb)\",\n",
" u'PR_NativeRunThread (in nss3.pdb)',\n",
" u'pr_root (in nss3.pdb)',\n",
" u'o__realloc_base (in ucrtbase.pdb)',\n",
" u'BaseThreadInitThunk (in kernel32.pdb)',\n",
" u'RtlUserThreadStart (in ntdll.pdb)',\n",
" u'BasepReportFault (in kernel32.pdb)',\n",
" u'BasepReportFault (in kernel32.pdb)']]),\n",
" (17175,\n",
" [[u'mozilla::a11y::DocAccessibleParent::CheckTopDoc() (in xul.pdb)',\n",
" u'mozilla::a11y::DocAccessibleParent::RecvShowEvent(mozilla::a11y::ShowEventData const &,bool const &) (in xul.pdb)',\n",
" u'mozilla::a11y::PDocAccessibleParent::OnMessageReceived(IPC::Message const &) (in xul.pdb)',\n",
" u'mozilla::dom::PContentParent::OnMessageReceived(IPC::Message const &) (in xul.pdb)',\n",
" u'mozilla::ipc::MessageChannel::DispatchAsyncMessage(IPC::Message const &) (in xul.pdb)',\n",
" u'mozilla::ipc::MessageChannel::DispatchMessageW(IPC::Message &&) (in xul.pdb)',\n",
" u'mozilla::ipc::MessageChannel::RunMessage(mozilla::ipc::MessageChannel::MessageTask &) (in xul.pdb)',\n",
" u'mozilla::ipc::MessageChannel::MessageTask::Run() (in xul.pdb)',\n",
" u'nsThread::ProcessNextEvent(bool,bool *) (in xul.pdb)',\n",
" u'NS_ProcessNextEvent(nsIThread *,bool) (in xul.pdb)',\n",
" u'mozilla::ipc::MessagePump::Run(base::MessagePump::Delegate *) (in xul.pdb)',\n",
" u'MessageLoop::RunHandler() (in xul.pdb)',\n",
" u'MessageLoop::Run() (in xul.pdb)',\n",
" u'nsBaseAppShell::Run() (in xul.pdb)',\n",
" u'nsAppShell::Run() (in xul.pdb)',\n",
" u'nsAppStartup::Run() (in xul.pdb)',\n",
" u'XREMain::XRE_mainRun() (in xul.pdb)',\n",
" u'XREMain::XRE_main(int,char * * const,mozilla::XREAppData const &) (in xul.pdb)',\n",
" u'XRE_main (in xul.pdb)',\n",
" u'do_main (in firefox.pdb)',\n",
" u'wmain (in firefox.pdb)',\n",
" u'__scrt_common_main_seh (in firefox.pdb)',\n",
" u'BaseThreadInitThunk (in kernel32.pdb)',\n",
" u'__RtlUserThreadStart (in ntdll.pdb)',\n",
" u'_RtlUserThreadStart (in ntdll.pdb)']])]"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"symbolicated_firsts.take(3)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"stack_counts = symbolicated_firsts.map(lambda pair: (str(pair[1]), 1)).countByKey()"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import operator\n",
"sorted_stack_counts = sorted(stack_counts.items(), key=operator.itemgetter(1), reverse=True)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"[('Expecting value: line 1 column 1 (char 0)', 103),\n",
" (\"'module_index'\", 82),\n",
" ('[[u\"mozilla::`anonymous namespace\\'::RunWatchdog (in xul.pdb)\", u\\'PR_NativeRunThread (in nss3.pdb)\\', u\\'pr_root (in nss3.pdb)\\', u\\'o__realloc_base (in ucrtbase.pdb)\\', u\\'BaseThreadInitThunk (in kernel32.pdb)\\', u\\'RtlUserThreadStart (in ntdll.pdb)\\', u\\'BasepReportFault (in kernel32.pdb)\\', u\\'BasepReportFault (in kernel32.pdb)\\']]',\n",
" 3),\n",
" (\"[[u'NS_QueryNotificationCallbacks<nsIChannel>(nsIChannel *,nsID const &,void * *) (in xul.pdb)', u'mozilla::net::HttpChannelParent::ConnectChannel(unsigned int const &,bool const &) (in xul.pdb)', u'mozilla::net::HttpChannelParent::Init(mozilla::net::HttpChannelCreationArgs const &) (in xul.pdb)', u'mozilla::net::NeckoParent::RecvPHttpChannelConstructor(mozilla::net::PHttpChannelParent *,mozilla::dom::PBrowserOrId const &,IPC::SerializedLoadContext const &,mozilla::net::HttpChannelCreationArgs const &) (in xul.pdb)', u'mozilla::net::PNeckoParent::OnMessageReceived(IPC::Message const &) (in xul.pdb)', u'mozilla::dom::PContentParent::OnMessageReceived(IPC::Message const &) (in xul.pdb)', u'mozilla::ipc::MessageChannel::DispatchAsyncMessage(IPC::Message const &) (in xul.pdb)', u'mozilla::ipc::MessageChannel::DispatchMessageW(IPC::Message &&) (in xul.pdb)', u'mozilla::ipc::MessageChannel::RunMessage(mozilla::ipc::MessageChannel::MessageTask &) (in xul.pdb)', u'mozilla::ipc::MessageChannel::MessageTask::Run() (in xul.pdb)', u'nsThread::ProcessNextEvent(bool,bool *) (in xul.pdb)', u'NS_ProcessNextEvent(nsIThread *,bool) (in xul.pdb)', u'mozilla::ipc::MessagePump::Run(base::MessagePump::Delegate *) (in xul.pdb)', u'MessageLoop::RunHandler() (in xul.pdb)', u'MessageLoop::Run() (in xul.pdb)', u'nsBaseAppShell::Run() (in xul.pdb)', u'nsAppShell::Run() (in xul.pdb)', u'nsAppStartup::Run() (in xul.pdb)', u'XREMain::XRE_mainRun() (in xul.pdb)', u'XREMain::XRE_main(int,char * * const,mozilla::XREAppData const &) (in xul.pdb)', u'XRE_main (in xul.pdb)', u'do_main (in firefox.pdb)', u'wmain (in firefox.pdb)', u'__scrt_common_main_seh (in firefox.pdb)', u'BaseThreadInitThunk (in kernel32.pdb)', u'__RtlUserThreadStart (in ntdll.pdb)', u'_RtlUserThreadStart (in ntdll.pdb)']]\",\n",
" 2),\n",
" ('[[u\"mozilla::`anonymous namespace\\'::RunWatchdog (in xul.pdb)\", u\\'PR_NativeRunThread (in nss3.pdb)\\', u\\'pr_root (in nss3.pdb)\\', u\\'o__realloc_base (in ucrtbase.pdb)\\', u\\'BaseThreadInitThunk (in kernel32.pdb)\\', u\\'RtlUserThreadStart (in ntdll.pdb)\\', u\\'GetWriteWatch (in kernelbase.pdb)\\']]',\n",
" 2),\n",
" (\"[[u'0x6a31 (in libobjc.A.dylib)', u'0x130d31 (in QuartzCore)', u'0x12420f (in QuartzCore)', u'0x12408d (in QuartzCore)', u'0x119877 (in QuartzCore)', u'0x10630 (in QuartzCore)', u'0x478444 (in AppKit)', u'0xa7396 (in CoreFoundation)', u'0xa7306 (in CoreFoundation)', u'0x87f38 (in CoreFoundation)', u'0x87973 (in CoreFoundation)', u'0x30acb (in HIToolbox)', u'0x30900 (in HIToolbox)', u'0x30735 (in HIToolbox)', u'0x46ae3 (in AppKit)', u'0x7c121e (in AppKit)', u'-[GeckoNSApplication nextEventMatchingMask:untilDate:inMode:dequeue:] (in XUL)', u'0x3b464 (in AppKit)', u'nsAppShell::Run() (in XUL)', u'nsAppStartup::Run() (in XUL)', u'XREMain::XRE_mainRun() (in XUL)', u'XREMain::XRE_main(int, char**, mozilla::XREAppData const&) (in XUL)', u'XRE_main (in XUL)', u'main (in firefox-bin)', u'start (in firefox-bin)']]\",\n",
" 2),\n",
" (\"[[u'0xe4fa (in libgobject-2.0.so.0.5000.2)', u'0xff74 (in libgobject-2.0.so.0.5000.2)', u'0x39118 (in libglib-2.0.so.0.5000.2)', u'0x2195b (in libgobject-2.0.so.0.5000.2)', u'0x3f0d7 (in libgobject-2.0.so.0.5000.2)', u'0x3b995 (in libgobject-2.0.so.0.5000.2)', u'0x4210f (in libgobject-2.0.so.0.5000.2)', u'0x25257f (in libgobject-2.0.so.0.5000.2)', u'0x2abcb (in libgobject-2.0.so.0.5000.2)', u'0x25257f (in libgobject-2.0.so.0.5000.2)', u'0x29fa1 (in libgobject-2.0.so.0.5000.2)', u'0x68342 (in libglib-2.0.so.0.5000.2)', u'0x4fe08 (in libglib-2.0.so.0.5000.2)', u'0x68342 (in libglib-2.0.so.0.5000.2)', u'0xdf6a (in libgobject-2.0.so.0.5000.2)', u'je_free (in firefox)', u'je_free (in firefox)', u'0x3d2137 (in libgtk-3.so.0.1400.5)', u'0x33601 (in libgobject-2.0.so.0.5000.2)', u'0x28d8f (in libgobject-2.0.so.0.5000.2)', u'0x25257f (in libgobject-2.0.so.0.5000.2)', u'0x2afae (in libgobject-2.0.so.0.5000.2)', u'je_free (in firefox)', u'0x25257f (in libgobject-2.0.so.0.5000.2)', u'0x4fec7 (in libglib-2.0.so.0.5000.2)', u'0xe5e70 (in libgtk-3.so.0.1400.5)', u'0xe64a3 (in libgtk-3.so.0.1400.5)', u'0x27fb05 (in libgtk-3.so.0.1400.5)', u'0x281f7f (in libgtk-3.so.0.1400.5)', u'0x2833ab (in libgtk-3.so.0.1400.5)', u'0x14c79 (in libgobject-2.0.so.0.5000.2)', u'nsLookAndFeel::Init() (in libxul.so)', u'nsChromeRegistry::QueryInterface(nsID const&, void**) (in libxul.so)', u'_fini (in libxul.so)', u'nsComponentManagerImpl::GetServiceByContractID(char const*, nsID const&, void**) (in libxul.so)', u'_fini (in libxul.so)', u'gfxFontStyle::gfxFontStyle() (in libxul.so)', u'mozilla::LookAndFeel::GetInt(mozilla::LookAndFeel::IntID, int*) (in libxul.so)', u'nsChromeRegistryChrome::CheckForOSAccessibility() (in libxul.so)', u'mozilla::services::GetToolkitChromeRegistryService() (in libxul.so)', u'_fini (in libxul.so)', u'ScopedXPCOMStartup::SetWindowCreator(nsINativeAppSupport*) (in libxul.so)', u'ProfileMissingDialog(nsINativeAppSupport*) (in libxul.so)', u'_fini (in libxul.so)', u'nsLocalFile::Release() (in libxul.so)', u'_fini (in libxul.so)', u'Output(bool, char const*, ...) [clone .constprop.69] (in libxul.so)', u'XRE_InitCommandLine (in libxul.so)', u'0x351c0 (in libc-2.24.so)', u'CheckArg(char const*, bool, char const**, bool) (in libxul.so)', u'XREMain::XRE_mainInit(bool*) (in libxul.so)', u'_fini (in libxul.so)', u'_fini (in libxul.so)', u'_fini (in libxul.so)', u'_fini (in libxul.so)', u'XREMain::XRE_main(int, char**, mozilla::XREAppData const&) (in libxul.so)', u'_fini (in libxul.so)', u'PR_Unlock (in libnspr4.so)', u'XRE_main (in libxul.so)', u'_fini (in libxul.so)', u'_fini (in libxul.so)', u'nsLocalFile::Append(nsAString_internal const&) (in libxul.so)', u'je_free (in firefox)', u'do_main(int, char**, char**, nsIFile*) (in firefox)', u'nsLocalFile::QueryInterface(nsID const&, void**) (in libxul.so)', u'NS_NewNativeLocalFile (in libxul.so)', u'_fini (in firefox)']]\",\n",
" 1),\n",
" (\"[[u'mozilla::ipc::FatalError(char const *,char const *,bool) (in xul.pdb)', u'mozilla::ipc::IProtocol::HandleFatalError(char const *,char const *) (in xul.pdb)', u'mozilla::ipc::IProtocol::FatalError(char const * const) (in xul.pdb)', u'mozilla::dom::PBrowserParent::OnMessageReceived(IPC::Message const &) (in xul.pdb)', u'mozilla::dom::PContentParent::OnMessageReceived(IPC::Message const &) (in xul.pdb)', u'mozilla::ipc::MessageChannel::DispatchAsyncMessage(IPC::Message const &) (in xul.pdb)', u'mozilla::ipc::MessageChannel::DispatchMessageW(IPC::Message &&) (in xul.pdb)', u'mozilla::ipc::MessageChannel::RunMessage(mozilla::ipc::MessageChannel::MessageTask &) (in xul.pdb)', u'mozilla::ipc::MessageChannel::MessageTask::Run() (in xul.pdb)', u'nsThread::ProcessNextEvent(bool,bool *) (in xul.pdb)', u'mozilla::ipc::MessagePump::Run(base::MessagePump::Delegate *) (in xul.pdb)', u'MessageLoop::RunHandler() (in xul.pdb)', u'MessageLoop::Run() (in xul.pdb)', u'nsBaseAppShell::Run() (in xul.pdb)', u'nsAppShell::Run() (in xul.pdb)', u'nsAppStartup::Run() (in xul.pdb)', u'XREMain::XRE_mainRun() (in xul.pdb)', u'XREMain::XRE_main(int,char * * const,mozilla::XREAppData const &) (in xul.pdb)', u'XRE_main (in xul.pdb)', u'do_main (in firefox.pdb)', u'wmain (in firefox.pdb)', u'__scrt_common_main_seh (in firefox.pdb)', u'BaseThreadInitThunk (in kernel32.pdb)', u'__RtlUserThreadStart (in ntdll.pdb)', u'_RtlUserThreadStart (in ntdll.pdb)']]\",\n",
" 1),\n",
" ('[[u\\'mozilla::a11y::DocAccessibleParent::RemoveChildDoc(mozilla::a11y::DocAccessibleParent *) (in xul.pdb)\\', u\"mozilla::a11y::DocAccessibleParent::`scalar deleting destructor\\'(unsigned int) (in xul.pdb)\", u\\'mozilla::a11y::DocAccessibleParent::Destroy() (in xul.pdb)\\', u\\'mozilla::dom::TabParent::DeallocPDocAccessibleParent(mozilla::a11y::PDocAccessibleParent *) (in xul.pdb)\\', u\\'mozilla::dom::PBrowserParent::RemoveManagee(int,mozilla::ipc::IProtocol *) (in xul.pdb)\\', u\\'mozilla::a11y::DocAccessibleParent::Destroy() (in xul.pdb)\\', u\\'mozilla::a11y::PDocAccessibleParent::Send__delete__(mozilla::a11y::PDocAccessibleParent *) (in xul.pdb)\\', u\\'mozilla::a11y::DocAccessibleParent::RecvShutdown() (in xul.pdb)\\', u\\'mozilla::a11y::PDocAccessibleParent::OnMessageReceived(IPC::Message const &) (in xul.pdb)\\', u\\'mozilla::dom::ContentParent::RecvScriptError(nsString const &,nsString const &,nsString const &,unsigned int const &,unsigned int const &,unsigned int const &,nsCString const &) (in xul.pdb)\\', u\\'stdext::hash_compare<int,std::less<int> >::operator()(int const &) (in xul.pdb)\\', u\\'free_impl (in mozglue.pdb)\\', u\\'mozilla::ipc::IToplevelProtocol::Lookup(int) (in xul.pdb)\\', u\\'fdlibm::scalbn(double,int) (in xul.pdb)\\', u\\'mozilla::dom::PContentParent::OnMessageReceived(IPC::Message const &) (in xul.pdb)\\', u\\'js::jit::InlineFrameIterator::resetOn(js::jit::JitFrameIterator const *) (in xul.pdb)\\']]',\n",
" 1),\n",
" (\"[[u'mozilla::a11y::DocAccessibleParent::CheckTopDoc() (in xul.pdb)', u'mozilla::a11y::DocAccessibleParent::RecvShowEvent(mozilla::a11y::ShowEventData const &,bool const &) (in xul.pdb)', u'mozilla::a11y::PDocAccessibleParent::OnMessageReceived(IPC::Message const &) (in xul.pdb)', u'mozilla::dom::PContentParent::OnMessageReceived(IPC::Message const &) (in xul.pdb)', u'mozilla::ipc::MessageChannel::DispatchAsyncMessage(IPC::Message const &) (in xul.pdb)', u'mozilla::ipc::MessageChannel::DispatchMessageW(IPC::Message &&) (in xul.pdb)', u'mozilla::ipc::MessageChannel::RunMessage(mozilla::ipc::MessageChannel::MessageTask &) (in xul.pdb)', u'mozilla::ipc::MessageChannel::MessageTask::Run() (in xul.pdb)', u'nsThread::ProcessNextEvent(bool,bool *) (in xul.pdb)', u'NS_ProcessNextEvent(nsIThread *,bool) (in xul.pdb)', u'mozilla::ipc::MessagePump::Run(base::MessagePump::Delegate *) (in xul.pdb)', u'MessageLoop::RunHandler() (in xul.pdb)', u'MessageLoop::Run() (in xul.pdb)', u'nsBaseAppShell::Run() (in xul.pdb)', u'nsAppShell::Run() (in xul.pdb)', u'nsAppStartup::Run() (in xul.pdb)', u'XREMain::XRE_mainRun() (in xul.pdb)', u'XREMain::XRE_main(int,char * * const,mozilla::XREAppData const &) (in xul.pdb)', u'XRE_main (in xul.pdb)', u'do_main (in firefox.pdb)', u'wmain (in firefox.pdb)', u'__scrt_common_main_seh (in firefox.pdb)', u'BaseThreadInitThunk (in kernel32.pdb)', u'__RtlUserThreadStart (in ntdll.pdb)', u'_RtlUserThreadStart (in ntdll.pdb)']]\",\n",
" 1)]"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sorted_stack_counts[:10]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Seeing a lot of shutdown crashes (RunWatchdog) which is exactly what I'd expect. Nice validation there.\n",
"\n",
"Things to do:\n",
"* figure out where `'Expecting value: line 1 column 1 (char 0)'` is coming from. It's from a json lib somewhere, but is it the request or the response or does it come from the server somehow?\n",
"* Determine if there's anything we can do with frames missing a module index (JIT frames, maybe?)."
]
}
],
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python [default]",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.12"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Raw
first stacks.py
# coding: utf-8
# ### When a user crashes for the first time, what is that crash?
# In[1]:
import ujson as json
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import plotly.plotly as py
import requests
from plotly.graph_objs import *
from moztelemetry import get_pings_properties, get_one_ping_per_client
from moztelemetry.dataset import Dataset
get_ipython().magic(u'matplotlib inline')
# In[2]:
sc.defaultParallelism
# In[3]:
pings = Dataset.from_source("telemetry") .where(docType='crash') .where(appUpdateChannel=lambda c: c in ('nightly')) .where(submissionDate=lambda d: d >= "20170101" and d <= "20170112") .records(sc, sample=1)
# In[4]:
subset = get_pings_properties(pings, [
"clientId",
"submissionDate",
"payload/stackTraces",
"environment/profile/creationDate"])\
.filter(lambda p: p["payload/stackTraces"] is not None)
# In[5]:
subset.count()
# First we need to filter out any profile older than our survey period.
# In[6]:
import datetime
survey_begin = (datetime.datetime(2017, 1, 1) - datetime.datetime(1970, 1, 1)).days
subset = subset.filter(lambda p: p["environment/profile/creationDate"] >= survey_begin)
# In[7]:
subset.count()
# So now we have a survey subset of pings with stack information. We want to get the first crash for each client reporting a crash, so...
# In[8]:
firsts = subset .map(lambda p: (p["clientId"], p)) .reduceByKey(lambda a, b: a if a["submissionDate"] < b["submissionDate"] else b) .map(lambda pair: pair[1])
# In[9]:
firsts.count()
# In[10]:
def symbolicate(s):
data = json.dumps({
'stacks': [[[f['module_index'], int(f['ip'], 16) - int(s['modules'][f['module_index']]['base_addr'], 16)] for f in s['threads'][s['crash_info']['crashing_thread']]['frames']]],
'memoryMap': [[m['debug_file'].translate(dict((ord(char), None) for char in ' ()')), m['debug_id']] for m in s['modules']], 'version': 4})
result = requests.post('http://symbolapi.mozilla.org/', data=data)
result_json = result.json()
return result_json['symbolicatedStacks']
# In[11]:
def safe_symbolicate(s):
try:
return symbolicate(s)
except Exception, e:
return e
# In[12]:
symbolicated_firsts = firsts.map(lambda p: (p["environment/profile/creationDate"], safe_symbolicate(p["payload/stackTraces"])))
# In[13]:
symbolicated_firsts.take(3)
# In[18]:
stack_counts = symbolicated_firsts.map(lambda pair: (str(pair[1]), 1)).countByKey()
# In[19]:
import operator
sorted_stack_counts = sorted(stack_counts.items(), key=operator.itemgetter(1), reverse=True)
# In[20]:
sorted_stack_counts[:10]
# Seeing a lot of shutdown crashes (RunWatchdog) which is exactly what I'd expect. Nice validation there.
#
# Things to do:
# * figure out where `'Expecting value: line 1 column 1 (char 0)'` is coming from. It's from a json lib somewhere, but is it the request or the response or does it come from the server somehow?
# * Determine if there's anything we can do with frames missing a module index (JIT frames, maybe?).
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment