Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import functions\n",
    "from datetime import datetime, timedelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_days = 1\n",
    "now = datetime.utcnow()\n",
    "days = [now - timedelta(days=i) for i in range(1, num_days + 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataset = SQLContext(sc).read.load(['s3://telemetry-parquet/socorro_crash/v1/crash_date=' + day.strftime('%Y%m%d') for day in days], 'parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "crashes = dataset\\\n",
    ".filter(dataset['product'] == 'Firefox')\\\n",
    ".select('signature', functions.explode(dataset['json_dump']['crashing_thread']['frames']).alias('frame')).rdd\\\n",
    ".filter(lambda r: r['frame']['file'] and 'media/webrtc' in r['frame']['file'])\\\n",
    ".groupBy(lambda r: r['signature'])\\\n",
    ".collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "90"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(crashes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'mozilla::dom::RTCCertificateBinding::Wrap',\n",
       " u'mozilla::dom::CallbackObject::CallSetup::CallSetup',\n",
       " u'WebRtcAgc_InitDigital',\n",
       " u'msvcr120.dll@0xf20c | webrtc::I420VideoFrame::CreateFrame',\n",
       " u'js::MarkAtoms',\n",
       " u'nr_ice_component_consent_timer_cb',\n",
       " u'moz_malloc_usable_size',\n",
       " u'CStdMarshal::DisconnectSrvIPIDs',\n",
       " u'mozilla::widget::NativeKey::GetFollowingCharMessage',\n",
       " u'OOM | large | mozalloc_abort | mozalloc_handle_oom | moz_xmalloc | mozilla::MakeUnique<T>',\n",
       " u'_wcsicmp',\n",
       " u'mozilla::PeerConnectionImpl::Initialize',\n",
       " u'moz_abort | pages_commit',\n",
       " u'js::InternalCallOrConstruct',\n",
       " u'vflydcam.ax@0x9650',\n",
       " u'js::TenuringTracer::traverse<T>',\n",
       " u'webrtc::DesktopAndCursorComposer::OnCaptureCompleted',\n",
       " u'GraphWalker<T>::DoWalk',\n",
       " u'nsNPAPIPluginInstance::Stop',\n",
       " u'webrtc::AudioDeviceWindowsCore::InitCaptureThreadPriority',\n",
       " u'RuleHash_ClassCSMatchEntry',\n",
       " u'AudioSrvBinding',\n",
       " u'hang | NtUserPeekMessage | NtUserPeekMessage | NtUserPeekMessage | mozilla::ipc::NeuteredWindowRegion::PumpOnce',\n",
       " u'nsCycleCollector::ScanWhiteNodes',\n",
       " u'nsACString_internal::Assign | nsACString_internal::Assign | mozilla::gmp::GetContentParentFromDone::Done',\n",
       " u'nsIDocument::WarnOnceAbout',\n",
       " u'webrtc::videocapturemodule::DeviceInfoDS::CreateCapabilityMap',\n",
       " u'Blt32to32BA_SSE2_Simple',\n",
       " u'js::jit::DoGetPropFallback',\n",
       " u'js::MapObject::mark',\n",
       " u'hang | npswf32_15_0_0_246.dll@0x6f3840',\n",
       " u'Abort | Could not initialize mScreenReferenceDrawTarget | mozalloc_abort | NS_DebugBreak | gfxPlatform::Init',\n",
       " u'IPCError-content | (msgtype=0x820017,name=???) Route error: message sent to unknown actor ID',\n",
       " u'nsTArray_Impl<T>::DestructRange | nsTArray_Impl<T>::RemoveElementsAt | mozilla::MediaEngineSource::Deallocate',\n",
       " u'mozilla::net::nsProtocolProxyService::LoadHostFilters',\n",
       " u\"`anonymous namespace''::CloseEventRunnable::WorkerRun\",\n",
       " u'webrtc::ThreadWindows::~ThreadWindows',\n",
       " u'memcpy | webrtc::I420VideoFrame::CreateFrame',\n",
       " u'qipcap.dll@0x31100',\n",
       " u'mozilla::WebrtcVideoConduit::SendPacket',\n",
       " u'IPCError-browser | PDocAccessibleParent::AddChildDoc binding to nonexistant proxy!',\n",
       " u'CPerChannelDbControl::GetDetails',\n",
       " u'nsCOMPtr_base::assign_from_gs_contractid_with_error | mozilla::DataChannelConnection::Init',\n",
       " u'IsAboutToBeFinalizedInternal<T>',\n",
       " u'memset | vp8_post_proc_frame',\n",
       " u'init_frame',\n",
       " u'mozilla::EverySecondTelemetryCallback_s',\n",
       " u'hang | PeekMessageW',\n",
       " u'CVideoControlInterfaceHandler::GetFrameRateList',\n",
       " u'nsCSSStyleSheet::GetOwningDocument',\n",
       " u'webrtc::AudioDeviceWindowsCore::DoCaptureThread',\n",
       " u'IPCError-browser | ShutDownKill',\n",
       " u'_moz_cairo_scaled_font_get_font_options',\n",
       " u'mozilla::WebrtcGmpVideoDecoder::ReceivedDecodedReferenceFrame',\n",
       " u'webrtc::ProcessThreadImpl::Process',\n",
       " u'mozilla::DataChannelConnection::Open',\n",
       " u'std::_Tree<T>::_Lbound',\n",
       " u'RGB24ToARGBRow_SSSE3',\n",
       " u'wsck32.dll@0x27263',\n",
       " u'OOM | large | mozalloc_abort | mozalloc_handle_oom | moz_xmalloc | mozilla::camera::DeliverFrameRunnable::DeliverFrameRunnable',\n",
       " u'js::GCMarker::lazilyMarkChildren',\n",
       " u'webrtc::Trace::SetTraceFile',\n",
       " u'webrtc::DelayManager::UpdateHistogram',\n",
       " u'nsRuleNode::GetStyleDisplay<T>',\n",
       " u'sipcc::EverySecondTelemetryCallback_s',\n",
       " u'OOM | large | mozalloc_abort | mozalloc_handle_oom | moz_xrealloc | nsTArray_base<T>::EnsureCapacity<T> | nsTArray_Impl<T>::AppendElements<T> | mozilla::MediaSegmentBase<T>::AppendFromInternal',\n",
       " u'nsRefPtr<T>::~nsRefPtr<T> | NativeInterfaceSweeper',\n",
       " u'general_composite_rect',\n",
       " u'lavsplitter.ax@0x28c11',\n",
       " u'CSFLog',\n",
       " u'OOM | small',\n",
       " u'JSScript::traceChildren',\n",
       " u'sdp_validate_sdp',\n",
       " u'nsRefPtr<T>::assign_with_AddRef | mozilla::net::nsHttpChannel::OnStopRequest',\n",
       " u'mozilla::dom::CreateInterfaceObjects',\n",
       " u'_PeekMessage',\n",
       " u'CSessMixer::GetSimpleVolume',\n",
       " u'js::IdIsIndex',\n",
       " u'zzz_AsmCodeRange_Begin | RtlpWaitOnCriticalSection | webrtc::CriticalSectionWindows::Leave',\n",
       " u'webrtc::VCMEncodedFrameCallback::Encoded',\n",
       " u\"mozilla::dom::indexedDB::`anonymous namespace''::DatabaseConnection::GetCachedStatement\",\n",
       " u'mozalloc_abort | NS_DebugBreak | mozilla::ipc::FatalError | mozilla::dom::PContentChild::SendPWebrtcGlobalConstructor',\n",
       " u'CNativeAudioParse::Init',\n",
       " u'js::ConstraintTypeSet::sweep',\n",
       " u'mozilla::SrtpFlow::ProtectRtcp',\n",
       " u'hang | NtUserMessageCall | RealDefWindowProcW',\n",
       " u'OOM | large | js::TraceRootRange<T>',\n",
       " u'PLDHashTable::Add | cft1st',\n",
       " u'OOM | large | mozalloc_abort | mozalloc_handle_oom | moz_xmalloc | mozilla::layers::BufferRecycleBin::GetBuffer',\n",
       " u'Interpret']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[c[0] for c in crashes]"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}