gistfile1.txt

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Splunk Magic</h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table class=\"nb_heading\">\n",
    "<tr>\n",
    "<td>\n",
    "    Project: Savvi @ NBN<br/>\n",
    "    Written: 12/12/2015<br/>\n",
    "    Author: Serge Rogov<br/>\n",
    "    Security: <span class=\"nb_sec nb_public\">Public</span><br/>\n",
    "</td>\n",
    "<td style=\"width: 50%\">\n",
    "    \n",
    "    <div class=\"savvi-logo\" style=\"height: 4em; background-position: right 0px\"></div>\n",
    "</td>\n",
    "</tr>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>\n",
       ".nb_heading{\n",
       "width: 100%;\n",
       "border: 0 !important;\n",
       "text-align: left;\n",
       "}\n",
       "\n",
       ".nb_heading tr{\n",
       "border: none;\n",
       "margin: 0;\n",
       "padding: 0;\n",
       "}\n",
       "\n",
       ".nb_heading td{\n",
       "border: none;\n",
       "margin: 0;\n",
       "padding: 0;\n",
       "}\n",
       "\n",
       ".nb_sec{\n",
       "border-radius: 0.2em;\n",
       "padding: 0.2em;\n",
       "}\n",
       "\n",
       ".nb_internal{\n",
       "background-color: red;\n",
       "color: white;\n",
       "}\n",
       "\n",
       ".nb_confidential{\n",
       "background-color: red;\n",
       "color: white;\n",
       "}\n",
       "\n",
       ".nb_public{\n",
       "background-color: hsl(111, 87%, 55%);\n",
       "}\n",
       "\n",
       "    \n",
       ".nb_message{\n",
       "    background-color: rgba(0, 24, 0, 0.05);\n",
       "    border-left: 0.3em solid gray;\n",
       "    border-radius: 0.15em;\n",
       "    padding: 0.2em;\n",
       "    padding-left: 0.3em;\n",
       "    margin-bottom: 0.2em;\n",
       "}\n",
       "\n",
       ".nb_message.nb_error{\n",
       "    background-color: rgba(255, 24, 77, 0.1);\n",
       "    border-left: 0.3em solid red;\n",
       "}\n",
       "\n",
       ".nb_message.nb_warning{\n",
       "    background-color: rgba(255, 184, 24, 0.2);\n",
       "    border-left: 0.3em solid orange !important;\n",
       "}\n",
       "    \n",
       ".savvi-logo {\n",
       "  background:       url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+Cjxzdmcgd2lkdGg9IjU4NHB4IiBoZWlnaHQ9IjE3MnB4IiB2aWV3Qm94PSIwIDAgNTg0IDE3MiIgdmVyc2lvbj0iMS4xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4bWxuczpza2V0Y2g9Imh0dHA6Ly93d3cuYm9oZW1pYW5jb2RpbmcuY29tL3NrZXRjaC9ucyI+CiAgICA8IS0tIEdlbmVyYXRvcjogU2tldGNoIDMuNCAoMTU1ODgpIC0gaHR0cDovL3d3dy5ib2hlbWlhbmNvZGluZy5jb20vc2tldGNoIC0tPgogICAgPHRpdGxlPnNhdnZpLWxvZ288L3RpdGxlPgogICAgPGRlc2M+Q3JlYXRlZCB3aXRoIFNrZXRjaC48L2Rlc2M+CiAgICA8ZGVmcz4KICAgICAgICA8cGF0aCBpZD0icGF0aC0xIiBkPSJNMC4wNiwwLjQwMiBMNTgzLjc4MSwwLjQwMiBMNTgzLjc4MSwxNzIgTDAuMDYsMTcyIj48L3BhdGg+CiAgICA8L2RlZnM+CiAgICA8ZyBpZD0iUGFnZS0xIiBzdHJva2U9Im5vbmUiIHN0cm9rZS13aWR0aD0iMSIgZmlsbD0ibm9uZSIgZmlsbC1ydWxlPSJldmVub2RkIiBza2V0Y2g6dHlwZT0iTVNQYWdlIj4KICAgICAgICA8ZyBpZD0ic2F2dmktbG9nbyIgc2tldGNoOnR5cGU9Ik1TTGF5ZXJHcm91cCI+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik01MDkuMDU3LDExOC45MTggQzUwNS4zMzEsMTE0LjIwNSA1MDUuMzMyLDEwNy41NzIgNTA5LjA1OCwxMDIuODYgTDQ5MC45NjEsODQuNzYyIEM0ODkuNDE5LDg1Ljk4MSA0ODcuNjYsODYuNzU3IDQ4NS44MzYsODcuMTg2IEw0ODUuODM2LDEzNC41OTIgQzQ4Ny42NTksMTM1LjAyMSA0ODkuNDE5LDEzNS43OTYgNDkwLjk2LDEzNy4wMTUgTDUwOS4wNTcsMTE4LjkxOCIgaWQ9IkZpbGwtMSIgZmlsbD0iI0RGNUUzMSIgc2tldGNoOnR5cGU9Ik1TU2hhcGVHcm91cCI+PC9wYXRoPgogICAgICAgICAgICA8cGF0aCBkPSJNNDkwLjI2LDE1Ny45MTIgQzQ4OS41MzMsMTU4LjQwOSA0ODguNzU5LDE1OC43OTMgNDg3Ljk2OSwxNTkuMTI1IEM0ODkuMzAxLDE2MS43NzggNDkxLjE5OCwxNjQuMDkgNDkzLjUxNCwxNjUuOTA2IEw0OTUuNDc2LDE2NS4xODMgTDQ5MC4yNiwxNTcuOTEyIiBpZD0iRmlsbC0yIiBmaWxsPSIjQzI0MTJEIiBza2V0Y2g6dHlwZT0iTVNTaGFwZUdyb3VwIj48L3BhdGg+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik01MzcuMjIxLDY0LjQzOSBDNTQyLjMyOSw2MC4zOTkgNTQ5LjczNSw2MC42OCA1NTQuNDUyLDY1LjM5OCBDNTU5LjE3LDcwLjExNSA1NTkuNDUxLDc3LjUyMSA1NTUuNDExLDgyLjYzIEw1NzMuNTA5LDEwMC43MjcgQzU3Ni40ODQsOTguMzc1IDU4MC4yMzIsOTcuNTM0IDU4My43ODEsOTguMTM1IEw1ODMuNzgxLDcxLjYgQzU4My43ODEsNjAuNTU5IDU3NC44Myw1MS42MDggNTYzLjc4OSw1MS42MDggTDUyNC4zOSw1MS42MDggTDUzNy4yMjEsNjQuNDM5IiBpZD0iRmlsbC0zIiBmaWxsPSIjRUI5NDJEIiBza2V0Y2g6dHlwZT0iTVNTaGFwZUdyb3VwIj48L3BhdGg+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik01MzIuMzQ2LDkzLjEwNiBMNTI3LjAxNiwxMDAuNTM3IEM1MjcuNDk3LDEwMC44OTkgNTI3Ljk4NSwxMDEuMjQ4IDUyOC40MjMsMTAxLjY4NiBDNTMzLjUwNSwxMDYuNzY5IDUzMy41MDUsMTE1LjAwOSA1MjguNDIzLDEyMC4wOTIgQzUyNy45ODUsMTIwLjUyOSA1MjcuNDk3LDEyMC44NzggNTI3LjAxNywxMjEuMjQgTDUzMi4zNDYsMTI4LjY3IEw1NjkuMjY3LDExNS4wODIgQzU2OC4zNDEsMTEyLjM2IDU2OC4zNDEsMTA5LjQxNiA1NjkuMjY4LDEwNi42OTQgTDUzMi4zNDYsOTMuMTA2IiBpZD0iRmlsbC00IiBmaWxsPSIjREY1RTMxIiBza2V0Y2g6dHlwZT0iTVNTaGFwZUdyb3VwIj48L3BhdGg+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik01MjYuNTQ3LDEwMC4xNTQgTDUzMS43NTcsOTIuODkgTDQ5NS4wMyw3OS4zNzQgQzQ5NC41NzMsODAuNTM0IDQ5My44OTYsODEuNjE0IDQ5My4wOTMsODIuNjMgTDUxMS4xOTEsMTAwLjcyNyBDNTE1LjY2Nyw5Ny4xODggNTIxLjg5MSw5Ni45NyA1MjYuNTQ3LDEwMC4xNTQiIGlkPSJGaWxsLTUiIGZpbGw9IiNFNzc5MzMiIHNrZXRjaDp0eXBlPSJNU1NoYXBlR3JvdXAiPjwvcGF0aD4KICAgICAgICAgICAgPHBhdGggZD0iTTUzMy4xNTEsNjkuODI3IEM1MzMuNjA4LDY4LjY2NiA1MzQuMjg1LDY3LjU4NyA1MzUuMDg4LDY2LjU3MSBMNTIwLjEyNSw1MS42MDggTDQ5OS43OTcsNTEuNjA4IEw0OTYuNDI0LDU2LjMxMSBMNTMzLjE1MSw2OS44MjciIGlkPSJGaWxsLTYiIGZpbGw9IiNFQjk0MkQiIHNrZXRjaDp0eXBlPSJNU1NoYXBlR3JvdXAiPjwvcGF0aD4KICAgICAgICAgICAgPHBhdGggZD0iTTUzMS43NTcsMTI4Ljg4NyBMNTI2LjU0NywxMjEuNjI0IEM1MjEuODkxLDEyNC44MDggNTE1LjY2NiwxMjQuNTkgNTExLjE5LDEyMS4wNSBMNDkzLjA5MywxMzkuMTQ3IEM0OTMuODk2LDE0MC4xNjMgNDk0LjU3MywxNDEuMjQzIDQ5NS4wMywxNDIuNDAzIEw1MzEuNzU3LDEyOC44ODciIGlkPSJGaWxsLTciIGZpbGw9IiNEODQ5MkMiIHNrZXRjaDp0eXBlPSJNU1NoYXBlR3JvdXAiPjwvcGF0aD4KICAgICAgICAgICAgPHBhdGggZD0iTTQ5MC43MjksNjQuMjUgQzQ5MS4yMDksNjQuNjEyIDQ5MS42OTcsNjQuOTYgNDkyLjEzNCw2NS4zOTggQzQ5NS43NjEsNjkuMDI0IDQ5Ni43NTEsNzQuMjQ3IDQ5NS4yMDIsNzguNzk0IEw1MzIuMTIyLDkyLjM4MSBMNTM3LjQ1Miw4NC45NTEgQzUzNi45NzIsODQuNTg5IDUzNi40ODQsODQuMjQxIDUzNi4wNDcsODMuODA0IEM1MzIuNDIsODAuMTc3IDUzMS40Myw3NC45NTQgNTMyLjk3OSw3MC40MDYgTDQ5Ni4wNTksNTYuODE5IEw0OTAuNzI5LDY0LjI1IiBpZD0iRmlsbC04IiBmaWxsPSIjRUI5NDJEIiBza2V0Y2g6dHlwZT0iTVNTaGFwZUdyb3VwIj48L3BhdGg+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik01NTMuMjc5LDg0Ljc2MiBDNTQ4LjgwMiw4OC4zMDIgNTQyLjU3Nyw4OC41MiA1MzcuOTIxLDg1LjMzNSBMNTMyLjcxMSw5Mi41OTggTDU2OS40NCwxMDYuMTE1IEM1NjkuODk3LDEwNC45NTQgNTcwLjU3MywxMDMuODc1IDU3MS4zNzcsMTAyLjg1OSBMNTUzLjI3OSw4NC43NjIiIGlkPSJGaWxsLTkiIGZpbGw9IiNFNzc5MzMiIHNrZXRjaDp0eXBlPSJNU1NoYXBlR3JvdXAiPjwvcGF0aD4KICAgICAgICAgICAgPHBhdGggZD0iTTU1My4yNzksMTM3LjAxNiBMNTcxLjM3NiwxMTguOTE4IEM1NzAuNTczLDExNy45MDIgNTY5Ljg5NiwxMTYuODIyIDU2OS40MzksMTE1LjY2MiBMNTMyLjcxMSwxMjkuMTc5IEw1MzcuOTIyLDEzNi40NDMgQzU0Mi41NzgsMTMzLjI1OCA1NDguODAyLDEzMy40NzYgNTUzLjI3OSwxMzcuMDE2IiBpZD0iRmlsbC0xMCIgZmlsbD0iI0Q4NDkyQyIgc2tldGNoOnR5cGU9Ik1TU2hhcGVHcm91cCI+PC9wYXRoPgogICAgICAgICAgICA8cGF0aCBkPSJNNDk1LjgzNSw1Ni4wOTQgTDQ5OS4wNTMsNTEuNjA4IEw0ODUuODM2LDUxLjYwOCBMNDg1LjgzNiw1Mi40MTUgTDQ5NS44MzUsNTYuMDk0IiBpZD0iRmlsbC0xMSIgZmlsbD0iI0VCOTQyRCIgc2tldGNoOnR5cGU9Ik1TU2hhcGVHcm91cCI+PC9wYXRoPgogICAgICAgICAgICA8cGF0aCBkPSJNNTczLjUwOCwxMjEuMDUgTDU1NS40MTEsMTM5LjE0OCBDNTU5LjQ1MSwxNDQuMjU2IDU1OS4xNywxNTEuNjYzIDU1NC40NTIsMTU2LjM4IEM1NDkuNzM1LDE2MS4wOTggNTQyLjMyOSwxNjEuMzc4IDUzNy4yMiwxNTcuMzM5IEw1MjQuMzksMTcwLjE3IEw1NjMuNzg5LDE3MC4xNyBDNTc0LjgzLDE3MC4xNyA1ODMuNzgxLDE2MS4yMTkgNTgzLjc4MSwxNTAuMTc4IEw1ODMuNzgxLDEyMy42NDMgQzU4MC4yMzIsMTI0LjI0NCA1NzYuNDgzLDEyMy40MDMgNTczLjUwOCwxMjEuMDUiIGlkPSJGaWxsLTEyIiBmaWxsPSIjQzI0MTJEIiBza2V0Y2g6dHlwZT0iTVNTaGFwZUdyb3VwIj48L3BhdGg+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik00ODUuODM2LDYxLjk3NiBDNDg3LjM4OCw2Mi4zMjcgNDg4Ljg5NCw2Mi45MzIgNDkwLjI2LDYzLjg2NiBMNDk1LjQ3LDU2LjYwMiBMNDg1LjgzNiw1My4wNTcgTDQ4NS44MzYsNjEuOTc2IiBpZD0iRmlsbC0xMyIgZmlsbD0iI0VCOTQyRCIgc2tldGNoOnR5cGU9Ik1TU2hhcGVHcm91cCI+PC9wYXRoPgogICAgICAgICAgICA8cGF0aCBkPSJNNTM3LjQ1MiwxMzYuODI2IEw1MzIuMTIyLDEyOS4zOTYgTDQ5NS4yMDIsMTQyLjk4MyBDNDk2Ljc1MSwxNDcuNTMxIDQ5NS43NjEsMTUyLjc1MyA0OTIuMTM0LDE1Ni4zOCBDNDkxLjY5NywxNTYuODE4IDQ5MS4yMDksMTU3LjE2NiA0OTAuNzI5LDE1Ny41MjggTDQ5Ni4wNjUsMTY0Ljk2NiBMNTMyLjk4LDE1MS4zNzMgQzUzMS40MywxNDYuODI1IDUzMi40MTksMTQxLjYwMiA1MzYuMDQ3LDEzNy45NzQgQzUzNi40ODQsMTM3LjUzNyA1MzYuOTcyLDEzNy4xODggNTM3LjQ1MiwxMzYuODI2IiBpZD0iRmlsbC0xNCIgZmlsbD0iI0MyNDEyRCIgc2tldGNoOnR5cGU9Ik1TU2hhcGVHcm91cCI+PC9wYXRoPgogICAgICAgICAgICA8cGF0aCBkPSJNNDk1Ljg0LDE2NS42OTIgTDQ5NC4wOSwxNjYuMzM2IEM0OTUuMjYsMTY3LjE4OCA0OTYuNTE4LDE2Ny45MjUgNDk3Ljg2MiwxNjguNTEgTDQ5NS44NCwxNjUuNjkyIiBpZD0iRmlsbC0xNSIgZmlsbD0iI0MyNDEyRCIgc2tldGNoOnR5cGU9Ik1TU2hhcGVHcm91cCI+PC9wYXRoPgogICAgICAgICAgICA8cGF0aCBkPSJNNTMzLjE1MSwxNTEuOTUzIEw0OTYuNDI5LDE2NS40NzUgTDQ5OC44OTMsMTY4LjkwOSBDNTAxLjA1NiwxNjkuNzEgNTAzLjM4NiwxNzAuMTcgNTA1LjgyOCwxNzAuMTcgTDUyMC4xMjUsMTcwLjE3IEw1MzUuMDg4LDE1NS4yMDcgQzUzNC4yODUsMTU0LjE5MSA1MzMuNjA4LDE1My4xMTIgNTMzLjE1MSwxNTEuOTUzIiBpZD0iRmlsbC0xNiIgZmlsbD0iI0MyNDEyRCIgc2tldGNoOnR5cGU9Ik1TU2hhcGVHcm91cCI+PC9wYXRoPgogICAgICAgICAgICA8cGF0aCBkPSJNMzA4LjIyLDgxLjM0NyBMMzE4LjQzOSw1MS45NDMgQzMxNi40MzQsNTEuMTA4IDMxMy45MjgsNTAuMzU2IDMxMC45Miw0OS42ODYgQzMwNy45MTMsNDkuMDIgMzA0LjczNiw0OC42ODQgMzAxLjM5NSw0OC42ODQgQzMwMC43Myw0OC42ODQgMzAwLjA5OSw0OC43MTkgMjk5LjQ2OSw0OC43NTUgQzI5OC42NzUsNDguOTY4IDI5Ny44NDIsNDguNjY5IDI5Ny4wNzUsNDguOTgiIGlkPSJGaWxsLTE3IiBmaWxsPSIjMEEwQTA4IiBza2V0Y2g6dHlwZT0iTVNTaGFwZUdyb3VwIj48L3BhdGg+CiAgICAgICAgICAgIDxnIGlkPSJHcm91cC0yMSI+CiAgICAgICAgICAgICAgICA8bWFzayBpZD0ibWFzay0yIiBza2V0Y2g6bmFtZT0iQ2xpcCAxOSIgZmlsbD0id2hpdGUiPgogICAgICAgICAgICAgICAgICAgIDx1c2UgeGxpbms6aHJlZj0iI3BhdGgtMSI+PC91c2U+CiAgICAgICAgICAgICAgICA8L21hc2s+CiAgICAgICAgICAgICAgICA8ZyBpZD0iQ2xpcC0xOSI+PC9nPgogICAgICAgICAgICAgICAgPHBhdGggZD0iTTI5Mi44MjMsNTEuMDI1IEwyOTIuNTE0LDUwLjE0MSBDMjg5Ljk0NSw1MS4xNDMgMjg3Ljg4OCw1Mi42NTggMjg2LjM1Nyw1NC42OTkgQzI4My4zNSw1OC43MDkgMjgwLjg0Myw2NC4zMDkgMjc4LjgzOSw3MS40OTMgTDI1OS43OSwxMzYuOTA5IEwyNDEuNzQzLDcyLjI0NCBDMjQwLjc0MSw2OC43MzUgMjM5LjY5NCw2NS41MjEgMjM4LjYxLDYyLjU5NCBDMjM3LjUyMSw1OS42NzMgMjM2LjEwMyw1Ny4yMDYgMjM0LjM0OSw1NS4yIEMyMzIuNTk1LDUzLjE5NSAyMzAuNDIyLDUxLjYwOSAyMjcuODMzLDUwLjQzOCBDMjI1LjI0LDQ5LjI3MSAyMjEuOTQzLDQ4LjY4NCAyMTcuOTMyLDQ4LjY4NCBDMjExLjI0Nyw0OC42ODQgMjA1LjIzMiw0OS43NzIgMTk5Ljg4Niw1MS45NDMgTDI0MS4yNDIsMTY5LjQ5MyBMMjc3LjU4NCwxNjkuNDkzIEwzMDUuODEyLDg4LjI3NSBMMjkyLjgyMyw1MS4wMjUiIGlkPSJGaWxsLTE4IiBmaWxsPSIjMEEwQTA4IiBza2V0Y2g6dHlwZT0iTVNTaGFwZUdyb3VwIiBtYXNrPSJ1cmwoI21hc2stMikiPjwvcGF0aD4KICAgICAgICAgICAgICAgIDxwYXRoIGQ9Ik05MS43MzQsNjMuNDcyIEM5MS43MzQsNjUuOTc5IDkwLjk0LDY4Ljk0NyA4OS4zNTMsNzIuMzY5IEM4Ny43NjQsNzUuNzk3IDg2LjA1Myw3OS4wMTEgODQuMjE2LDgyLjAxOSBDODIuNzExLDgwLjg1MiA4MC43MDcsNzkuNTEyIDc4LjIwMSw3OC4wMDkgQzc1LjY5NCw3Ni41MDUgNzIuOTM2LDc1LjEyNiA2OS45Myw3My44NzQgQzY2LjkyMSw3Mi42MjEgNjMuNzQ1LDcxLjU3OSA2MC40MDUsNzAuNzQxIEM1Ny4wNiw2OS45MDYgNTMuODAyLDY5LjQ4NyA1MC42Myw2OS40ODcgQzQ4Ljc4OSw2OS40ODcgNDYuODcsNjkuNjEzIDQ0Ljg2Niw2OS44NjMgQzQyLjg2LDcwLjExNCA0MS4wNjMsNzAuNjE1IDM5LjQ3Niw3MS4zNjcgQzM3Ljg4Nyw3Mi4xMTggMzYuNTUxLDczLjIwOCAzNS40NjYsNzQuNjI1IEMzNC4zNzgsNzYuMDQ3IDMzLjgzNiw3Ny45MjcgMzMuODM2LDgwLjI2NSBDMzMuODM2LDg1Ljk0NyAzOS45MzUsOTAuMTI2IDUyLjEzMyw5Mi43OTYgQzU3LjE0Niw5My43OTkgNjIuMTE2LDk1LjA5NiA2Ny4wNDcsOTYuNjgxIEM3MS45NzQsOTguMjcyIDc2LjQwMywxMDAuNDg0IDgwLjMzLDEwMy4zMjMgQzg0LjI1NSwxMDYuMTY3IDg3LjQzLDEwOS43NTggODkuODU1LDExNC4xMDIgQzkyLjI3NiwxMTguNDQ5IDkzLjQ5LDEyMy44NzcgOTMuNDksMTMwLjM5MyBDOTMuNDksMTM3LjA3OCA5Mi4xOTMsMTQzLjAxMSA4OS42MDQsMTQ4LjE4OSBDODcuMDEyLDE1My4zNjkgODMuNDY0LDE1Ny43MTMgNzguOTUzLDE2MS4yMjMgQzc0LjQ0MSwxNjQuNzMgNjkuMTM1LDE2Ny40MDUgNjMuMDM3LDE2OS4yNDIgQzU2LjkzNiwxNzEuMDc5IDUwLjI5NCwxNzIgNDMuMTEsMTcyIEMzNS41OTIsMTcyIDI5LjMyNSwxNzEuNDU2IDI0LjMxMiwxNzAuMzcgQzE5LjI5OSwxNjkuMjgyIDE1LjEyMSwxNjcuOTg5IDExLjc4MSwxNjYuNDg2IEM5LjQzOCwxNjUuNDgyIDYuODkzLDE2NC4wMjIgNC4xMzcsMTYyLjA5OSBDMS4zNzksMTYwLjE4IDAsMTU3LjU0OSAwLDE1NC4yMDQgQzAsMTUxLjM2NSAwLjgzNSwxNDguMzk5IDIuNTA3LDE0NS4zMDcgQzQuMTc1LDE0Mi4yMTUgNi4wOTgsMTM5LjAwMSA4LjI3MiwxMzUuNjU2IEM5Ljc3NSwxMzYuOTk1IDExLjY1NSwxMzguMzc0IDEzLjkxMSwxMzkuNzkxIEMxNi4xNjcsMTQxLjIxMyAxOC43NTUsMTQyLjU0OSAyMS42OCwxNDMuODAyIEMyNC42MDMsMTQ1LjA1NSAyNy43MzYsMTQ2LjEwMSAzMS4wOCwxNDYuOTM0IEMzNC40MjEsMTQ3Ljc3NCAzNy44NDcsMTQ4LjE4OSA0MS4zNTYsMTQ4LjE4OSBDNDMuNTI1LDE0OC4xODkgNDUuNzgyLDE0OC4wMjQgNDguMTIzLDE0Ny42ODggQzUwLjQ2MSwxNDcuMzU0IDUyLjUxLDE0Ni42ODQgNTQuMjY0LDE0NS42ODIgQzU2LjAxOCwxNDQuNjc5IDU3LjQ4LDE0My4zNDQgNTguNjUsMTQxLjY3MSBDNTkuODE4LDE0MC4wMDMgNjAuNDA1LDEzNy44MyA2MC40MDUsMTM1LjE1NSBDNjAuNDA1LDEzMC4zMSA1OC41MjUsMTI2Ljg0NSA1NC43NjUsMTI0Ljc1MyBDNTEuMDA1LDEyMi42NjUgNDUuMzY3LDEyMC44NjggMzcuODQ3LDExOS4zNjUgQzMyLjk5OSwxMTguMzYxIDI4LjMyMywxMTcuMDI3IDIzLjgxMSwxMTUuMzU1IEMxOS4yOTksMTEzLjY4NyAxNS4yOSwxMTEuNDcgMTEuNzgxLDEwOC43MTIgQzguMjcyLDEwNS45NTUgNS40NzEsMTAyLjQ5IDMuMzg1LDk4LjMxIEMxLjI5Myw5NC4xMzcgMC4yNTEsODkuMDM2IDAuMjUxLDgzLjAyMSBDMC4yNTEsNzcuMDA2IDEuNTg2LDcxLjc0MyA0LjI2MSw2Ny4yMzEgQzYuOTMzLDYyLjcyIDEwLjQ4NSw1OC44NzggMTQuOTE0LDU1LjcwMyBDMTkuMzM5LDUyLjUzIDI0LjQ3Nyw1MC4xNDkgMzAuMzI5LDQ4LjU1OSBDMzYuMTc1LDQ2Ljk3MyA0Mi4yNzMsNDYuMTc4IDQ4LjYyNCw0Ni4xNzggQzU1LjMwNiw0Ni4xNzggNjEuMjgzLDQ2LjY3OSA2Ni41NDYsNDcuNjgyIEM3MS44MSw0OC42ODQgNzYuMzYsNTAuMDIzIDgwLjIwNiw1MS42OTIgQzg3LjU1Niw1NC42OTkgOTEuMzk4LDU4LjYyOCA5MS43MzQsNjMuNDcyIiBpZD0iRmlsbC0yMCIgZmlsbD0iIzBBMEEwOCIgc2tldGNoOnR5cGU9Ik1TU2hhcGVHcm91cCIgbWFzaz0idXJsKCNtYXNrLTIpIj48L3BhdGg+CiAgICAgICAgICAgIDwvZz4KICAgICAgICAgICAgPHBhdGggZD0iTTE3MS4zNzYsMTEwLjA5MSBMMTY3Ljg2NiwxMTAuMDkxIEMxNjYuNjk1LDExMC4wOTEgMTY1LjUyNCwxMTAuMTc3IDE2NC4zNTcsMTEwLjM0MiBDMTYwLjE3OSwxMTAuNjc4IDE1Ni4wNDQsMTExLjIxOCAxNTEuOTUsMTExLjk3MSBDMTQ3Ljg1NCwxMTIuNzIzIDE0NC4yMiwxMTMuODk0IDE0MS4wNDksMTE1LjQ3OSBDMTM3Ljg3MSwxMTcuMDcgMTM1LjI4MywxMTkuMjAxIDEzMy4yNzgsMTIxLjg3MSBDMTMxLjI3NCwxMjQuNTQ1IDEzMC4yNywxMjcuOTczIDEzMC4yNywxMzIuMTQ3IEMxMzAuMjcsMTM2LjgyNyAxMzEuNjA2LDE0MC43OTUgMTM0LjI4MSwxNDQuMDUyIEMxMzYuOTUyLDE0Ny4zMTEgMTQxLjQ2NCwxNDguOTQgMTQ3LjgxNSwxNDguOTQgQzE1MC45ODcsMTQ4Ljk0IDE1My45OTUsMTQ4LjUyNSAxNTYuODM5LDE0Ny42ODggQzE1OS42NzgsMTQ2Ljg1MyAxNjIuMTg0LDE0NS43NjggMTY0LjM1NywxNDQuNDI5IEMxNjYuNTI4LDE0My4wOTMgMTY4LjI0MywxNDEuNjMzIDE2OS40OTYsMTQwLjA0MiBDMTcwLjc0OSwxMzguNDU3IDE3MS4zNzYsMTM2Ljk5NSAxNzEuMzc2LDEzNS42NTYgTDE3MS4zNzYsMTEwLjA5MSBMMTcxLjM3NiwxMTAuMDkxIFogTTIwMy40NTcsMTM5LjE2NiBDMjAzLjQ1NywxNTIuMTk4IDIwNS4zNzcsMTYxLjgxIDIwOS4yMjMsMTY3Ljk4OSBDMjA2LjIxNSwxNjkuMTU2IDIwMy40MTQsMTY5Ljk5NCAyMDAuODI1LDE3MC40OTUgQzE5OC4yMzQsMTcwLjk5NyAxOTUuNTE5LDE3MS4yNDggMTkyLjY4LDE3MS4yNDggQzE4Ni42NjUsMTcxLjI0OCAxODIuMTEsMTcwLjA3NyAxNzkuMDIsMTY3LjczOSBDMTc1LjkyNiwxNjUuNDAxIDE3My45NjQsMTYyLjA2IDE3My4xMywxNTcuNzEzIEMxNjkuNDUzLDE2MS41NTkgMTY0LjYwOCwxNjQuODE2IDE1OC41OTMsMTY3LjQ4OCBDMTUyLjU3OCwxNzAuMTU4IDE0NS4yMjIsMTcxLjQ5OSAxMzYuNTM3LDE3MS40OTkgQzEzMi4wMjUsMTcxLjQ5OSAxMjcuNTE0LDE3MC44NzEgMTIzLjAwMiwxNjkuNjE5IEMxMTguNDkxLDE2OC4zNjYgMTE0LjM5NCwxNjYuMzIxIDExMC43MiwxNjMuNDc4IEMxMDcuMDQzLDE2MC42MzggMTA0LjAzNSwxNTYuODM1IDEwMS42OTcsMTUyLjA3MyBDOTkuMzU2LDE0Ny4zMTEgOTguMTg5LDE0MS41MDcgOTguMTg5LDEzNC42NTQgQzk4LjE4OSwxMjYuMTMyIDEwMC4yMzMsMTE5LjIwMSAxMDQuMzI5LDExMy44NTEgQzEwOC40MjIsMTA4LjUwNSAxMTMuNTU5LDEwNC4yODcgMTE5Ljc0MywxMDEuMTkzIEMxMjUuOTIzLDk4LjEwMyAxMzIuNjA4LDk1Ljk3MiAxMzkuNzk0LDk0LjgwMiBDMTQ2Ljk3OCw5My42MzQgMTUzLjU4LDkyLjg4MiAxNTkuNTk1LDkyLjU0NiBDMTYxLjYwMSw5Mi4zODEgMTYzLjU2Miw5Mi4yOTUgMTY1LjQ4NSw5Mi4yOTUgTDE3MC44NzUsOTIuMjk1IEwxNzAuODc1LDg3Ljc4NCBDMTcwLjg3NSw4MS40MzUgMTY5LjE1OSw3Ni45MjMgMTY1LjczNiw3NC4yNDkgQzE2Mi4zMSw3MS41NzkgMTU2Ljc1Myw3MC4yNCAxNDkuMDY4LDcwLjI0IEMxNDEuODgyLDcwLjI0IDEzNS40OSw3MS40MSAxMjkuODk1LDczLjc0OCBDMTI0LjI5NCw3Ni4wOSAxMTguODIzLDc4Ljc2IDExMy40NzgsODEuNzY5IEMxMTEuMTM1LDc4LjkyOSAxMDkuMjU1LDc1Ljc5NyAxMDcuODM4LDcyLjM2OSBDMTA2LjQxNiw2OC45NDcgMTA1LjcwNyw2Ni4zOTcgMTA1LjcwNyw2NC43MjQgQzEwNS43MDcsNjIuNTU1IDEwNyw2MC4zMzkgMTA5LjU5Myw1OC4wODQgQzExMi4xODIsNTUuODI3IDExNS43MzMsNTMuODIzIDEyMC4yNDQsNTIuMDY3IEMxMjQuNzU2LDUwLjMxMyAxMjkuOTMzLDQ4Ljg5NiAxMzUuNzg0LDQ3LjgwNiBDMTQxLjYzMSw0Ni43MjIgMTQ3LjgxNSw0Ni4xNzggMTU0LjMzMiw0Ni4xNzggQzE2My4zNTUsNDYuMTc4IDE3MC45OTksNDcuMDk3IDE3Ny4yNjYsNDguOTM0IEMxODMuNTMyLDUwLjc3NiAxODguNjI2LDUzLjYxNSAxOTIuNTU1LDU3LjQ1NyBDMTk2LjQ3OCw2MS4zMDIgMTk5LjI3OSw2Ni4xMDMgMjAwLjk1MSw3MS44NjggQzIwMi42MTksNzcuNjM0IDIwMy40NTcsODQuMzYyIDIwMy40NTcsOTIuMDQ1IEwyMDMuNDU3LDEzOS4xNjYgTDIwMy40NTcsMTM5LjE2NiBaIiBpZD0iRmlsbC0yMiIgZmlsbD0iIzBBMEEwOCIgc2tldGNoOnR5cGU9Ik1TU2hhcGVHcm91cCI+PC9wYXRoPgogICAgICAgICAgICA8cGF0aCBkPSJNNDU1LjQxNCwxOC42OTkgQzQ1NS40MTQsMjQuMDQ4IDQ1My42MTcsMjguMzkyIDQ1MC4wMjYsMzEuNzMzIEM0NDYuNDMxLDM1LjA3OCA0NDIuMTMxLDM2Ljc0NiA0MzcuMTE4LDM2Ljc0NiBDNDMxLjkzNywzNi43NDYgNDI3LjU5NCwzNS4wNzggNDI0LjA4NSwzMS43MzMgQzQyMC41NzUsMjguMzkyIDQxOC44MjEsMjQuMDQ4IDQxOC44MjEsMTguNjk5IEM0MTguODIxLDEzLjM1MyA0MjAuNTc1LDguOTY3IDQyNC4wODUsNS41NDEgQzQyNy41OTQsMi4xMTcgNDMxLjkzNywwLjQwMiA0MzcuMTE4LDAuNDAyIEM0NDIuMTMxLDAuNDAyIDQ0Ni40MzEsMi4xMTcgNDUwLjAyNiw1LjU0MSBDNDUzLjYxNyw4Ljk2NyA0NTUuNDE0LDEzLjM1MyA0NTUuNDE0LDE4LjY5OSIgaWQ9IkZpbGwtMjMiIGZpbGw9IiMwQTBBMDgiIHNrZXRjaDp0eXBlPSJNU1NoYXBlR3JvdXAiPjwvcGF0aD4KICAgICAgICAgICAgPHBhdGggZD0iTTQ1NS4xNjQsMTY5Ljc0MyBDNDU0LjQ5NSwxNzAuMDc3IDQ1Mi45OSwxNzAuMzI3IDQ1MC42NTIsMTcwLjQ5NSBDNDQ4LjMxMSwxNzAuNjU5IDQ0NS44MDQsMTcwLjc0NyA0NDMuMTMzLDE3MC43NDcgQzQzNi42MTcsMTcwLjc0NyA0MzEuMzEsMTY5LjM2OCA0MjcuMjE3LDE2Ni42MSBDNDIzLjEyMSwxNjMuODU0IDQyMS4wNzYsMTU4LjMgNDIxLjA3NiwxNDkuOTQzIEw0MjEuMDc2LDQ5LjY4NiBDNDIxLjQxLDQ5LjUyMiA0MjIuNzQ2LDQ5LjMxMSA0MjUuMDg3LDQ5LjA2IEM0MjcuNDI1LDQ4LjgxIDQyOS45MzIsNDguNjg0IDQzMi42MDcsNDguNjg0IEM0MzUuMjc3LDQ4LjY4NCA0MzcuOTUyLDQ4LjkzNCA0NDAuNjI2LDQ5LjQzNiBDNDQzLjI5OCw0OS45MzcgNDQ1LjcyMiw1MC45IDQ0Ny44OTYsNTIuMzE4IEM0NTAuMDY1LDUzLjc0IDQ1MS44Miw1NS43NDYgNDUzLjE1OSw1OC4zMzQgQzQ1NC40OTUsNjAuOTI2IDQ1NS4xNjQsNjQuMzA5IDQ1NS4xNjQsNjguNDg0IEw0NTUuMTY0LDE2OS43NDMiIGlkPSJGaWxsLTI0IiBmaWxsPSIjMEEwQTA4IiBza2V0Y2g6dHlwZT0iTVNTaGFwZUdyb3VwIj48L3BhdGg+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yOTcuMDYsNDguOTg3IEMzMDUuNjQ3LDQ1LjU4OSAzMTcuMTY2LDQ2LjY0NSAzMjUuNTQ4LDUwLjQzOCBDMzI4LjEzNyw1MS42MDkgMzMwLjMxLDUzLjE5NSAzMzIuMDY0LDU1LjIgQzMzMy44MTgsNTcuMjA2IDMzNS4yMzUsNTkuNjczIDMzNi4zMjUsNjIuNTk0IEMzMzcuNDA5LDY1LjUyMSAzMzguNDU2LDY4LjczNSAzMzkuNDU4LDcyLjI0NCBMMzU3LjUwNSwxMzYuOTA5IEwzNzYuNTUzLDcxLjQ5MyBDMzc4LjU1OCw2NC4zMDkgMzgxLjA2NSw1OC43MDkgMzg0LjA3Miw1NC42OTkgQzM4Ny4wOCw1MC42OSAzOTIuMDkzLDQ4LjY4NCAzOTkuMTEsNDguNjg0IEM0MDIuNDUsNDguNjg0IDQwNS42MjgsNDkuMDIgNDA4LjYzNCw0OS42ODYgQzQxMS42NDMsNTAuMzU2IDQxNC4xNDgsNTEuMTA4IDQxNi4xNTQsNTEuOTQzIEwzNzUuMjk5LDE2OS40OTMgTDMzOC45NTcsMTY5LjQ5MyBDMzM2Ljc5NiwxNjMuMjk1IDMzNC42MzUsMTU3LjA5OCAzMzIuNDc0LDE1MC45MDEgQzMyNy43MTMsMTM3LjI0OCAzMjIuOTUyLDEyMy41OTYgMzE4LjE5MiwxMDkuOTQzIEMzMTMuNDA2LDk2LjIxOSAzMDguNjIsODIuNDk1IDMwMy44MzUsNjguNzcxIiBpZD0iRmlsbC0yNSIgZmlsbD0iIzBBMEEwOCIgc2tldGNoOnR5cGU9Ik1TU2hhcGVHcm91cCI+PC9wYXRoPgogICAgICAgIDwvZz4KICAgIDwvZz4KPC9zdmc+)\n",
       "    no-repeat\n",
       "    left center;\n",
       "  background-size: contain;\n",
       "}    \n",
       "\n",
       "</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import time\n",
    "import io\n",
    "import httplib2\n",
    "from IPython.display import display, HTML \n",
    "import splunklib.results as results\n",
    "import splunklib.client\n",
    "import json\n",
    "import IPython.display\n",
    "from IPython.core.magic import (register_line_magic, \n",
    "                                register_cell_magic)\n",
    "import qgrid2 as qgrid\n",
    "\n",
    "\n",
    "plt.style.use('ggplot')\n",
    "with open('custom_html.html', 'r') as f:\n",
    "    custom_html = f.read()\n",
    "display(HTML(custom_html))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from splunk_nb import *\n",
    "if sys.version_info[0] < 3:\n",
    "    from StringIO import StringIO\n",
    "else:\n",
    "    from io import StringIO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('splunk-auth-yong', 'r') as f:\n",
    "    #TODO SR: encrypt splunk auth? \n",
    "    splunk_auth = json.loads(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "service = splunklib.client.connect(autologin=True, **splunk_auth) #app=\"apm_snpm\","
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def replace_splunk_time(df):\n",
    "    '''Converts `nb_epoch` into `_time` as datetime64\n",
    "        `nb_epoch=_time` must be supplied by the query\n",
    "\n",
    "    '''\n",
    "    if ('nb_epoch' not in df.columns):\n",
    "        return\n",
    "    \n",
    "    df['_time'] = pd.to_numeric(df['nb_epoch'], errors='coerce').astype('datetime64[s]')\n",
    "    df.drop('nb_epoch', 1, inplace=True)\n",
    "\n",
    "class SplunkQuery:\n",
    "    job = None\n",
    "    search_string = None\n",
    "    def __init__(self, search_string):\n",
    "        if not search_string.startswith('search '):\n",
    "            search_string = 'search ' + search_string + ' | eval nb_epoch=_time | fields - _time'\n",
    "        self.search_string = search_string\n",
    "        \n",
    "    def _dispatch_query(self):\n",
    "        self.job = service.jobs.create(self.search_string, **{\"exec_mode\": \"normal\", \n",
    "                                      \"earliest_time\": '-10y', \n",
    "                                      \"latest_time\": '-0min',\n",
    "                                      \"output_mode\": \"csv\",\n",
    "                                      \"preview\": True,\n",
    "                                      \"maxEvents\": 0})\n",
    "    \n",
    "    def _await(self):\n",
    "        while True:\n",
    "            self.job.refresh()\n",
    "            if self.job[\"isDone\"] == \"1\":\n",
    "                break\n",
    "            time.sleep(1)\n",
    "    \n",
    "    def _report_progress(self):\n",
    "        #publish progress (stdout, NB)\n",
    "        pass\n",
    "    \n",
    "    def _await_with_progress(self):\n",
    "        print \"waiting\"\n",
    "        while True:\n",
    "            self.job.refresh()\n",
    "            if self.job[\"isDone\"] == \"1\":\n",
    "                break\n",
    "            time.sleep(1)\n",
    "        print \"done\"\n",
    "        \n",
    "    def _df_postprocess(self, df):\n",
    "        replace_splunk_time(df)\n",
    "        return df\n",
    "        \n",
    "    def _get_results_legacy(self):\n",
    "        \"\"\"Fetches one page of results using the offically recommended\n",
    "        approach. This method is SLOW.\n",
    "         \n",
    "            Args:\n",
    "                offset: start offset\n",
    "                count: number of results to return\n",
    "        \"\"\"\n",
    "        job = self.job\n",
    "        resultCount = job[\"resultCount\"]  # Number of results this job returned\n",
    "        offset = 0;                       # Start at result 0\n",
    "        count = 100;                       # Get sets of 10 results at a time\n",
    "        items = []\n",
    "        \n",
    "        while (offset < int(resultCount)):\n",
    "            kwargs_paginate = {\"count\": count,\n",
    "                        \"offset\": offset}\n",
    "            # Get the search results and display them\n",
    "            blocksearch_results = job.preview(output_mode=\"csv\", **kwargs_paginate)\n",
    "\n",
    "            for result in results.ResultsReader(blocksearch_results):\n",
    "                items.append(result)\n",
    "            offset += count\n",
    "        df = pd.DataFrame(items)\n",
    "        df = self._df_postprocess(df)\n",
    "        return df\n",
    "    \n",
    "    def get_preview(self):\n",
    "        self.job.refresh()\n",
    "        buf = StringIO()\n",
    "        job = self.job\n",
    "        self.buf=buf\n",
    "        \n",
    "        if (self.job['dispatchState']=='PARSING'):\n",
    "            return None #haven't received a resultPreviewCount yet TODO: backport to other cases\n",
    "        resultCount = int(self.job['resultPreviewCount'])\n",
    "        if (resultCount==0):\n",
    "            return None #no preview yet\n",
    "        \n",
    "        offset = 0\n",
    "        page_count = 1000\n",
    "        \n",
    "        while (offset < resultCount):\n",
    "            kwargs_paginate = {\"count\": page_count,\n",
    "                        \"offset\": offset}\n",
    "            \n",
    "            searchresults = job.preview(output_mode=\"csv\", **kwargs_paginate).read()  \n",
    "            \n",
    "            #suppress the CSV header on pages other than the first\n",
    "            if (offset == 0):\n",
    "                 buf.write(searchresults)\n",
    "            else:\n",
    "                 buf.write(searchresults[searchresults.find('\\n'):])\n",
    "            offset+=page_count\n",
    "        \n",
    "        buf.seek(0)\n",
    "        \n",
    "        df = pd.read_csv(buf)\n",
    "        df = self._df_postprocess(df)\n",
    "        return df\n",
    "    \n",
    "    def display_messages(self):\n",
    "        classes = {'info': 'nb_info', 'fatal': 'nb_error', 'error': 'nb_error'}\n",
    "        html = ''\n",
    "        for k, v in self.job.messages.iteritems():\n",
    "            line = '<div class=\"nb_message {classname}\">{message}</div>'.format(classname=classes[k], message=v[0])\n",
    "            html = html + line\n",
    "        display(HTML(html))\n",
    "    \n",
    "    def _get_results_page_fast(self, offset, count):\n",
    "        \"\"\"Fetches one page of results\n",
    "         \n",
    "            Args:\n",
    "                offset: start offset\n",
    "                count: number of results to return\n",
    "        \"\"\"\n",
    "        #TODO: uses `service` which is in global scope. Refactor\n",
    "        buf = StringIO()\n",
    "        self.buf=buf\n",
    "        sid = self.job['sid']\n",
    "        myhttp = httplib2.Http(disable_ssl_certificate_validation=True)\n",
    "        myhttp.add_credentials(service.username, service.password)\n",
    "        url = '/services/search/jobs/{0}/results?output_mode=csv&&offset={1}&count={2}'.format(sid, offset, count)\n",
    "        baseurl = str(service.authority)\n",
    "        searchresults = myhttp.request(baseurl + url, 'GET')[1]        \n",
    "        buf.write(searchresults)\n",
    "        \n",
    "        buf.seek(0)\n",
    "        df = pd.read_csv(buf)\n",
    "        df = self._df_postprocess(df)\n",
    "        return df\n",
    "    \n",
    "    def _get_results_fast_full(self, page_count=50000):\n",
    "        \"\"\"Fetches entire result set quickly\n",
    "         \n",
    "            Args:\n",
    "                offset: start offset\n",
    "                page_count: maximum number of results per page (default splunk limit is 50k)\n",
    "            \n",
    "            Notes:\n",
    "                Not sure which Splunk setting dictates maximum number of\n",
    "                results returned (page count). Ideally should identify it\n",
    "                and dynamically read via SDK\n",
    "        \"\"\"\n",
    "        #TODO: uses `service` which is in global scope. Refactor\n",
    "        buf = StringIO()\n",
    "        self.buf=buf\n",
    "        sid = self.job['sid']\n",
    "        resultCount = int(self.job['resultCount'])\n",
    "        \n",
    "        myhttp = httplib2.Http(disable_ssl_certificate_validation=True)\n",
    "        myhttp.add_credentials(service.username, service.password)\n",
    "        \n",
    "        offset = 0\n",
    "        \n",
    "        while (offset < resultCount):\n",
    "            url = '/services/search/jobs/{0}/results?output_mode=csv&&offset={1}&count={2}'.format(\n",
    "                sid, offset, page_count)\n",
    "            \n",
    "            baseurl = str(service.authority)\n",
    "            searchresults = myhttp.request(baseurl + url, 'GET')[1]   \n",
    "            \n",
    "            #suppress the CSV header on pages other than the first\n",
    "            if (offset == 0):\n",
    "                 buf.write(searchresults)\n",
    "            else:\n",
    "                 buf.write(searchresults[searchresults.find('\\n'):])\n",
    "            offset+=page_count\n",
    "        \n",
    "        buf.seek(0)\n",
    "        df = pd.read_csv(buf)\n",
    "        df = self._df_postprocess(df)\n",
    "        return df\n",
    "    \n",
    "        \n",
    "        \n",
    "    def execute(self, **kwargs):\n",
    "        \"\"\"Executes the query\n",
    "         \n",
    "            Args:\n",
    "                TODO: add args\n",
    "        \"\"\"\n",
    "        self._dispatch_query()\n",
    "        #self._await_with_progress()\n",
    "        return\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def run_blocking(query):\n",
    "    global last_job\n",
    "    x = SplunkQuery(query)\n",
    "    last_job = x\n",
    "    x.execute()\n",
    "    x._await()\n",
    "    return x._get_results_fast_full()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def preview_kernel(df):\n",
    "    chart=df.set_index('_time')['count'].astype('float')\n",
    "    plt.gca().cla() \n",
    "    chart.plot()\n",
    "    IPython.display.clear_output(wait=True)\n",
    "    IPython.display.display(plt.gcf()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def run_preview(query):\n",
    "    global last_job\n",
    "    x = SplunkQuery(query)\n",
    "    last_job = x\n",
    "    x.execute()\n",
    "    while(x.job.is_done() == False):\n",
    "        \n",
    "        d = x.get_preview()\n",
    "        if (d is None):\n",
    "            IPython.display.clear_output(wait=True)\n",
    "            print \"waiting\"\n",
    "            sys.stdout.flush()\n",
    "            continue\n",
    "        \n",
    "        preview_kernel(d)\n",
    "        \n",
    "        time.sleep(1.0)\n",
    "    \n",
    "    IPython.display.clear_output(wait=True)\n",
    "    x.display_messages()\n",
    "    print \"Done!\"\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def run_preview(query):\n",
    "    global last_job\n",
    "    x = SplunkQuery(query)\n",
    "    last_job = x\n",
    "    first_results = True\n",
    "    x.execute()\n",
    "    while(x.job.is_done() == False):\n",
    "        \n",
    "        d = x.get_preview()\n",
    "        if (d is None):\n",
    "            IPython.display.clear_output(wait=True)\n",
    "            print x.job['dispatchState']\n",
    "            sys.stdout.flush()\n",
    "            time.sleep(1.0)\n",
    "            continue\n",
    "        else:\n",
    "            if (first_results):\n",
    "                grid = qgrid.QGridWidget(df=d)\n",
    "                display(grid)\n",
    "            first_results = False\n",
    "        \n",
    "        grid.df = d\n",
    "    \n",
    "        #preview_kernel(d)\n",
    "        \n",
    "        time.sleep(1.0)\n",
    "    \n",
    "    IPython.display.clear_output(wait=False)\n",
    "    x.display_messages()\n",
    "    print \"Done!\"\n",
    "    return x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "@register_cell_magic\n",
    "def splunk(line, cell):\n",
    "    query = cell\n",
    "    if('preview=True' in line):\n",
    "        run_preview(query)\n",
    "    else:\n",
    "        return run_blocking(query)\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x = run_preview('source=\"megadump_60.tgz:*\" earliest=\"11/30/2015:20:00:00\" | timechart span=4h avg(max_latency) as count')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<br/>\n",
    "<br/>\n",
    "<br/>\n",
    "<br/>\n",
    "<br/>\n",
    "<br/>\n",
    "<br/>\n",
    "<br/>\n",
    "<br/>\n",
    "<br/>\n",
    "<br/>\n",
    "<br/>\n",
    "<br/>\n",
    "<br/>\n",
    "<br/>\n",
    "<br/>\n",
    "<br/>\n",
    "<br/>\n",
    "<br/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%splunk preview=True\n",
    "source=\"megadump_60.tgz:*\" earliest=\"12/04/2015:20:00:00\" | timechart span=4h avg(max_latency) as count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df=last_job._get_results_fast_full()\n",
    "df.set_index('_time')['count'].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}