SGE+Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SGE vs. GRACC Comparison\n",
    "\n",
    "In this notebook, we compare one day from an SGE accounting log with the same day in GRACC.\n",
    "\n",
    "## Results\n",
    "\n",
    "The SGE accounting log and GRACC agree on the vast majority of jobs.  A large number of short jobs are not collected in the GRACC accounting.  The total core hours missing from GRACC is 166.95, out of 207,418.51 total core hours for the day.  0.08% core hours missing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculations starting: 2019-04-14 00:00:00+00:00\n",
      "Calculations ending: 2019-04-14 23:59:59+00:00\n"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "import dateutil.parser\n",
    "start_time = dateutil.parser.parse(\"2019-04-14 00:00:00 GMT\")\n",
    "end_time = dateutil.parser.parse(\"2019-04-14 23:59:59 GMT\")\n",
    "print(\"Calculations starting:\", start_time)\n",
    "print(\"Calculations ending:\", end_time)\n",
    "start_time_timestamp = int(start_time.strftime(\"%s\"))\n",
    "end_time_timestamp = int(end_time.strftime(\"%s\"))\n",
    "\n",
    "# Read in the first 10 lines for beginning of analysis\n",
    "sge_accounting_f = open('accounting', 'r')\n",
    "sge_lines = []\n",
    "# Toss the first 4 lines, it's always a comment\n",
    "for i in range(4):\n",
    "    sge_accounting_f.readline()\n",
    "\n",
    "# Field names\n",
    "sgeFields = [ \"qname\",\n",
    "                      \"hostname\",\n",
    "                      \"group\",\n",
    "                      \"owner\",\n",
    "                      \"job_name\",\n",
    "                      \"job_number\",\n",
    "                      \"account\",\n",
    "                      \"priority\",\n",
    "                      \"submission_time\",\n",
    "                      \"start_time\",\n",
    "                      \"end_time\",\n",
    "                      \"failed\",\n",
    "                      \"exit_status\",\n",
    "                      \"ru_wallclock\",\n",
    "                      \"ru_utime\",\n",
    "                      \"ru_stime\",\n",
    "                      \"ru_maxrss\",\n",
    "                      \"ru_ixrss\",\n",
    "                      \"ru_ismrss\",\n",
    "                      \"ru_idrss\",\n",
    "                      \"ru_isrss\",\n",
    "                      \"ru_minflt\",\n",
    "                      \"ru_majflt\",\n",
    "                      \"ru_nswap\",\n",
    "                      \"ru_inblock\",\n",
    "                      \"ru_oublock\",\n",
    "                      \"ru_msgsnd\",\n",
    "                      \"ru_msgrcv\",\n",
    "                      \"ru_nsignals\",\n",
    "                      \"ru_nvcsw\",\n",
    "                      \"ru_nivcsw\",\n",
    "                      \"project\",\n",
    "                      \"department\",\n",
    "                      \"granted_pe\",\n",
    "                      \"slots\",\n",
    "                      \"task_number\",\n",
    "                      \"cpu\",\n",
    "                      \"mem\",\n",
    "                      \"io\",\n",
    "                      \"category\",\n",
    "                      \"iow\",\n",
    "                      \"pe_taskid\",\n",
    "                      \"maxvmem\" ]\n",
    "\n",
    "\n",
    "jobids = {}\n",
    "# Load in all the job ids into a dictionary, with value being the cpu\n",
    "for line in sge_accounting_f:\n",
    "    sgeList = line.rstrip(\"\\r\\n\").split(\":\")\n",
    "    sgeRecord = dict(zip(sgeFields, sgeList))\n",
    "    if int(sgeRecord['end_time']) < start_time_timestamp or int(sgeRecord['end_time']) > end_time_timestamp:\n",
    "        continue\n",
    "    if sgeRecord['job_number'] in jobids:\n",
    "        print(\"Duplicate jobid!!!\")\n",
    "    # Calculate core hours\n",
    "    core_hours = (float(sgeRecord['ru_wallclock']) / 3600.0) * float(sgeRecord['slots'])\n",
    "    jobids[sgeRecord['job_number']] = (core_hours, sgeRecord)\n",
    "sge_accounting_f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate from GRACC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/derekweitzel/.local/lib/python3.6/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.25.2) or chardet (3.0.4) doesn't match a supported version!\n",
      "  RequestsDependencyWarning)\n"
     ]
    }
   ],
   "source": [
    "!pip install -U -q elasticsearch elasticsearch_dsl\n",
    "import elasticsearch\n",
    "from elasticsearch_dsl import Search, Q\n",
    "import urllib3\n",
    "import math\n",
    "\n",
    "# Get rid of insecure warning\n",
    "urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\n",
    "\n",
    "import ssl\n",
    "from elasticsearch.connection import create_ssl_context\n",
    "\n",
    "ssl_context = create_ssl_context(cafile = \"/usr/local/etc/openssl/cert.pem\")\n",
    "ssl_context.check_hostname = False\n",
    "ssl_context.verify_mode = ssl.CERT_NONE\n",
    "\n",
    "\n",
    "es = elasticsearch.Elasticsearch(\n",
    "        ['https://gracc.opensciencegrid.org/q'],\n",
    "        timeout=300, use_ssl=True,  ssl_context=ssl_context)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "osg_summary_index = 'gracc.osg.raw-*'\n",
    "s = Search(using=es, index=osg_summary_index)\n",
    "probe_name = 'sge:atlas-ce.bu.edu'\n",
    "\n",
    "s = s.query('bool',\n",
    "        filter=[\n",
    "            Q('range', EndTime={'gt': start_time + datetime.timedelta(hours=6), 'lt': end_time + datetime.timedelta(hours=6)})\n",
    "          & Q('term', ProbeName=probe_name)\n",
    "          & Q('term', ResourceType='Batch')\n",
    "        ]\n",
    "    )\n",
    "bkt = s.aggs\n",
    "bkt = bkt.metric('WallDuration',       'sum', field='WallDuration')\n",
    "bkt = bkt.metric('NumberOfJobs',       'sum', field='Count')\n",
    "bkt = bkt.metric('EarliestEndTime',    'min', field='EndTime')\n",
    "bkt = bkt.metric('LatestEndTime',      'max', field='EndTime')\n",
    "bkt = bkt.metric('CoreHours',          'sum', field='CoreHours')\n",
    "result = s.execute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Earliest EndTime: 2019-04-14T06:00:02.000Z\n",
      "Last EndTime: 2019-04-15T05:59:56.000Z\n",
      "Number of jobs: 31821\n",
      "Wall Duration in Seconds: 298675129.0\n",
      "Core Hours: 207418.51194443685\n"
     ]
    }
   ],
   "source": [
    "print(\"Earliest EndTime: {}\".format(result.aggregations.EarliestEndTime.value_as_string))\n",
    "print(\"Last EndTime: {}\".format(result.aggregations.LatestEndTime.value_as_string))\n",
    "print(\"Number of jobs: {}\".format(result.hits.total))\n",
    "print(\"Wall Duration in Seconds: {}\".format(result.aggregations.WallDuration.value))\n",
    "print(\"Core Hours: {}\".format(result.aggregations.CoreHours.value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Jobs that were found in SGE but not in GRACC: 11662\n",
      "Jobs found in GRACC but not in SGE: 0\n"
     ]
    }
   ],
   "source": [
    "missing_jobs = []\n",
    "tmp_jobids = dict(jobids)\n",
    "for hit in s.scan():\n",
    "    if hit['LocalJobId'] in tmp_jobids:\n",
    "        if math.isclose(hit['CoreHours'],tmp_jobids[hit['LocalJobId']][0]):\n",
    "            del tmp_jobids[hit['LocalJobId']]\n",
    "        else:\n",
    "            print(\"Jobids match, but corehours do not\", hit['LocalJobId'], hit['CoreHours'], tmp_jobids[hit['LocalJobId']])\n",
    "    else:\n",
    "      #print(\"JobId not found in condor: {}\".format(hit['LocalJobId']))\n",
    "      #print(\"ES: {}, HTCondor: {}\".format(hit['CoreHours'], tmp_jobids[hit['LocalJobId']]))\n",
    "      missing_jobs.append(hit)\n",
    "print(\"Jobs that were found in SGE but not in GRACC: {}\".format(str(len(tmp_jobids))))\n",
    "print(\"Jobs found in GRACC but not in SGE: {}\".format(len(missing_jobs)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Core hours in SGE and not in GRACC: 166.9561111111062\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.dates as mdates\n",
    "%matplotlib inline\n",
    "import matplotlib.font_manager\n",
    "\n",
    "sge_jobs = []\n",
    "missing_core_hours = 0\n",
    "for k, v in tmp_jobids.items():\n",
    "    sge_jobs.append([datetime.datetime.fromtimestamp(int(v[1]['end_time'])), v[0]])\n",
    "    missing_core_hours += v[0]\n",
    "print(\"Core hours in SGE and not in GRACC:\", missing_core_hours)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Graph of missing core hours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEJCAYAAACHRBAhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XuYXXV97/H3J8MEB0SjMKQwJAY1pqJcQkdCkSOgRaBq\nSb0cQfBWNQcfeU4Rn5wDRyqt1cop9crFNKUcStHgLcRYAxGvKBpkwi0GSBsRISOSQAggREkm3/PH\nWjvs7OzLmpm9Zu295vN6nnmy12+ttef3y5rZ3/ndFRGYmZm1MqXoDJiZWXdwwDAzs0wcMMzMLBMH\nDDMzy8QBw8zMMnHAMDOzTBwwzMwsEwcMMzPLxAHDzMwy2aPoDLTTfvvtF7NmzSo6G2ZmXWX16tWP\nRER/q+tKFTBmzZrF0NBQ0dkwM+sqkn6d5To3SZmZWSYOGGZmlkmuAUPSDEk/kHS3pLWS/rrONZL0\nBUnrJd0l6ciqcydLWpeeOy/PvJqZWXN51zC2Ax+JiEOAo4EPSTqk5ppTgNnp1wLgiwCSeoDL0vOH\nAKfXudfMzCZIrgEjIh6KiNvS108C9wADNZedClwdiVXANEkHAEcB6yPivoh4Brg2vdbMzAowYaOk\nJM0C5gK31JwaAB6sOt6QptVLn1fnfReQ1EyYOXNm2/JrZvVdsGwNS255kJEIeiROnzeDT8w/tOhs\n2QSYkE5vSc8FvgGcExFPtPO9I2JxRAxGxGB/f8thxGY2DhcsW8M1qx5gJN2pcySCa1Y9wAXL1hSc\nM5sIuQcMSb0kweJLEbG0ziXDwIyq44PStEbpZlaQJbc8OKp0K5e8R0kJ+Ffgnoj4TIPLlgPvSkdL\nHQ08HhEPAbcCsyUdLGkqcFp6rZkVpFKzyJpu5ZJ3H8argXcCayTdkab9H2AmQEQsAlYAfw6sB54G\n3pue2y7pbGAl0ANcGRFrc86vmTXRI9UNDj1SAbmxiZZrwIiInwBNf5IiIoAPNTi3giSgmFkHOH3e\nDK5Z9UDddCu/Uq0lZWb5qoyG8iipyclLg5hZZh5SO7m5hmFmmVSG1FZUhtQCDhqThGsYZpbJl+r0\nXTRLt/JxwDCzTBoNnPWA2snDAcPMzDJxwDAzs0wcMMwsk1e/5IWjSrfyccAws0zufujJUaVb+Thg\nmFkmjz29bVTpVj4OGGZmlokDhpll0mhROC87OHk4YJhZJp6HYbkuDSLpSuCNwMaIeGWd8wuBM6ry\n8nKgPyI2S7ofeBIYAbZHxGCeeTUzs+byrmFcBZzc6GREXBwRR0TEEcD5wI8iYnPVJSek5x0szMwK\nlmvAiIibgM0tL0ycDizJMTtmNg6z9997VOlWPh3RhyFpL5KayDeqkgP4rqTVkhY0uXeBpCFJQ5s2\nbco7q2aT1v777Fk3fd6L953gnFhROiJgAG8Cbq5pjjo2bao6BfiQpNfUuzEiFkfEYEQM9vf3T0Re\nzSadZbcPc/Mv6zcWLLnlwQnOjRWlUwLGadQ0R0XEcPrvRuA64KgC8mVmwMUr1zU8V2+PbyunwgOG\npOcDxwHfrErbW9I+ldfA64FfFJNDM/vNlq0Nz/XIMzEmi7yH1S4Bjgf2k7QBuBDoBYiIRellfwl8\nJyKeqrp1OnCdkh/EPYAvR8QNeebVzBqbtldvwyVATp83Y4JzY0XJNWBExOkZrrmKZPhtddp9wOH5\n5MrMRusP20bqpk/B27NOJoU3SZlZ53t624666fVTrawcMMzMLBMHDDNryQsPGjhgmFkGXnjQwAHD\nzDJoNHTWQ2onFwcMM2up0eQ8T9qbXBwwzKylgWl9o0q3cnLAMLOWFp40h77enl3S+np7WHjSnIJy\nZEXIdeKemZXD/LkDQLKm1G+2bOXAaX0sPGnOznSbHBwwzCyT+XMHHCAmOTdJmZlZJg4YZmaWiQOG\nmZll4oBhZmaZ5BowJF0paaOkupsfSTpe0uOS7ki/PlZ17mRJ6yStl3Renvk0M7PW8q5hXAWc3OKa\nH0fEEenXxwEk9QCXkeznfQhwuqRDcs2pmZk1lWvAiIibgPo7xzd3FLA+Iu6LiGeAa4FT25o5MzMb\nlU7owzhG0l2Srpf0ijRtAHiw6poNadpuJC2QNCRpaNOmTXnn1cxs0io6YNwGzIyIw4BLgGWjfYOI\nWBwRgxEx2N/f3/YMmplZotCAERFPRMTv0tcrgF5J+wHDQPXO8gelaWZmVpBCA4akP5KSBfUlHZXm\n51HgVmC2pIMlTQVOA5YXl1MzM8t1LSlJS4Djgf0kbQAuBHoBImIR8Fbgg5K2A1uB0yIigO2SzgZW\nAj3AlRGxNs+8mplZc4oSbYAyODgYQ0NDRWfDzKyrSFodEYOtriu609vMzLqEA4aZmWXigGFmZpk4\nYJiZWSYOGGZmlokDhpmZZeKAYWZmmThgmJlZJg4YZmaWiQOGmZll4oBhZmaZOGCYmVkmDhhmZpZJ\nrgFD0pWSNkr6RYPzZ6Tbs66R9FNJh1eduz9Nv0OSl6A1MytYpoAh6R8lPU9Sr6TvSdok6cwMt14F\nnNzk/K+A4yLiUODvgcU150+IiCOyLLtrZmb5ylrDeH1EPAG8EbgfeCmwsNVNEXETsLnJ+Z9GxGPp\n4SqSrVjNzKwDZQ0Yvem/bwC+FhGP55CX9wHXVx0H8F1JqyUtaHSTpAWShiQNbdq0KYdsmZkZZN+i\ndbmke0m2Uf2gpH7g9+3KhKQTSALGsVXJx0bEsKT9gRsl3ZvWWHYREYtJm7IGBwfLs32gmVmHaVnD\nkDQF+BZwDDAYEduAp4FT25EBSYcBVwCnRsSjlfSIGE7/3QhcBxzVju9nZmZj0zJgRMQO4LKI2BwR\nI2naUxHx2/F+c0kzgaXAOyPiP6vS95a0T+U18Hqg7kgrMzObGFmbpL4n6S3A0ojI3OwjaQlwPLCf\npA3AhaT9IRGxCPgYsC9wuSSA7emIqOnAdWnaHsCXI+KGrN/XzMzaT1k+/yU9CewNbCfpuxAQEfG8\nfLM3OoODgzE05CkbZmajIWl1lukLmWoYEbHP+LNkZmbdLFPAkPSaeun1Ri2ZmVk5Ze3DqJ6k9xyS\nEUurgde2PUdmZtaRsjZJvan6WNIM4HO55MjMzDrSWBcf3AC8vJ0ZMTOzzpa1D+MSkqU6IAkyRwC3\n5ZUpMzPrPFn7MKrHqm4HlkTEzTnkx8zMOlTWPox/kzQVeFmatC6/LJmZWSfK2iR1PPBvJEubC5gh\n6d0eVmtmNnlkbZL6NMmeGOsAJL0MWAL8SV4ZMzOzzpJ5P4xKsABIFwrsbXK9mZmVTOZOb0lXANek\nx2ewa0e4mZmVXNaA8UHgQ8D/TI9/DFyeS47MzKwjZWqSiog/RMRnIuLN6ddnI+IPre6TdKWkjZLq\n7mWhxBckrZd0l6Qjq86dLGldeu687EUyM7M8NK1hSFrDsxP2dhMRh7V4/6uAS4GrG5w/BZidfs0D\nvgjMk9QDXAacSDKr/FZJyyPi7hbfz8zMctKqSeqN6b8Cvg38+WjePCJukjSrySWnAlenmzKtkjRN\n0gHALGB9RNwHIOna9FoHDDOzgjQNGBHx68prSX+oPm6TAeDBquMNaVq99Hn13kDSAmABwMyZM9uc\nPTMzqxjr4oMdIyIWR8RgRAz29/cXnR0zs9Jq1YdxZNVhn6S5JM1TAETEeBcgHAZmVB0flKb1Nkg3\nM7OCtOrD+HTV698Cn6k6Dsa/gdJy4Oy0j2Ie8HhEPCRpEzBb0sEkgeI04B3j/F5mZjYOrfowThjP\nm0taAhwP7CdpA3Ah6QzxiFgErCDpSF8PPA28Nz23XdLZwEqgB7gyItaOJy9mZjY+WSfujUlEnN7i\nfJBMCKx3bgVJQDEzsw7Q9Z3eZmY2MRwwzMwsk0wBI13C40xJH0uPZ0o6Kt+smZlZJ8law7gc+FOg\n0ifxJMnSHWZmNklk7fSeFxFHSrodICIeS7dsNbOSWnb7MBevXMdvtmzlwGl9LDxpDvPnDhSdLStQ\n1oCxLV0QMAAk9QM7csuVmRVq2e3DnL90DVu3jQAwvGUr5y9dA+CgMYllbZL6AnAdsL+kTwI/Af4h\nt1yZWaEuXrluZ7Co2LpthItXrmtwh00GmWoYEfElSauB15EsDTI/Iu7JNWdmVpjhLVtHlW6TQ8uA\nkTZFrY2IPwbuzT9LZla0HomR2H0rnB6pztU2WbRskoqIEWCdJK8dbjZJ1AsWzdJtcsja6f0CYK2k\nnwNPVRIj4i9yyZWZFWpgWl/d5qeBaX0F5MY6RdaA8Te55sLMOsrCk+bsMkoKoK+3h4UnzSkwV1a0\nrJ3eP5I0HXhVmvTziNiYX7bMrEiVobOeh2HVMgUMSf8duBj4IckoqUskLYyIr+eYNzMr0Py5Aw4Q\ntousTVIfBV5VqVWkE/e+CzQNGJJOBj5PsqfFFRFxUc35hcAZVXl5OdAfEZsl3U+yBMkIsD0iBjPm\n1czMcpA1YEypaYJ6lBYjrNLhuJcBJwIbgFslLY+IuyvXRMTFJDUXJL0J+HBEbK56mxMi4pGMeTQz\nsxxlDRg3SFoJLEmP3w5c3+Keo4D1EXEfQLoN66nA3Q2uP73q/c3MrMNkWhokIhYC/wwcln4tjoj/\n1eK2AeDBquMNadpuJO0FnAx8o/rbAt+VtFrSgkbfRNICSUOShjZt2tS6MGZmNiZNaxiSXgpMj4ib\nI2IpsDRNP1bSSyLil23Kx5uAm2uao46NiGFJ+wM3Sro3Im6qvTEiFgOLAQYHBz2ryMwsJ61qGJ8D\nnqiT/nh6rplhYEbV8UFpWj2nUdMcFRHD6b8bSRY+9IZNZmYFahUwpkfEmtrENG1Wi3tvBWZLOjjd\nO+M0YHntRZKeDxwHfLMqbW9J+1ReA68HftHi+5mZWY5adXpPa3Ku6RoBEbFd0tnASpJhtVdGxFpJ\nZ6XnF6WX/iXwnYh4qur26cB1ShY62wP4ckTc0CKvZmaWI0WTxcQkLQG+HxH/UpP+fuDEiHh7zvkb\nlcHBwRgaGio6G2ZmXUXS6ixz3VrVMM4h+Uv/DGB1mjYITCWpGZiZ2STRNGBExMPAMZJOAF6ZJn87\nIr6fe87MrDDez9vqybr44A+AH+ScFzPrAN7P2xrJuqe3mU0S3s/bGnHAMLNd/KbBvt2N0m3ycMAw\ns10c2GBXvUbpNnk4YJjZLhaeNIe+3p5d0rzbnkH21WrNbJLwbnvWiAOGme3Gu+1ZPW6SMjOzTBww\nzMwsEwcMMzPLxAHDzMwyyTVgSDpZ0jpJ6yWdV+f88ZIel3RH+vWxrPeamdnEym2UlKQe4DLgRJL9\nvG+VtDwi7q659McR8cYx3mtmZhMkzxrGUcD6iLgvIp4BrgVOnYB7zcwsB3kGjAHgwarjDWlarWMk\n3SXpekmvGOW9SFogaUjS0KZNm9qRbzMzq6PoTu/bgJkRcRhwCbBstG8QEYsjYjAiBvv7+9ueQTMz\nS+Q503sYmFF1fFCatlNEPFH1eoWkyyXtl+Ves8nCmxlZp8gzYNwKzJZ0MMmH/WnAO6ovkPRHwMMR\nEZKOIqnxPApsaXWv2WSw7PZhzv3KHexIj4e3bOXcr9wBdO5mRg5w5ZVbwIiI7ZLOBlYCPcCVEbFW\n0lnp+UXAW4EPStoObAVOi4gA6t6bV17NOtX5S+/aGSwqdqTpnfgh7N36yi3XxQcjYgWwoiZtUdXr\nS4FLs95rNtls3VYbLpqnF63Zbn0OGN2v6E5vMysR79ZXbl7e3KwLKef3H2s/xLS9enns6W27pXu3\nvnJwwDDrUMtubzwwMHL+vmPph1h2+zC/+/323dJ7e+Td+krCTVJmHerilesK+76N+iFa3bdtx+6h\nbO+pe7j/oiQcMMw61HBB7f5j7YdodP7xrbs3UVl3csAw61BT8u6oaGDaXr1101v1QzQ67/6L8nDA\nMOtQdVp3durN6Td3PP0QC0+aQ19vzy5pfb097r8oEQcMsy6U1zSM8fRDzJ87wKfefCjT+p6toTwn\nr8hmhfAoKbMaZ/zLz7j5l5t3Hr/6JS/kSx/40wJztLu8mqva0Q/x1B+eraE89vQ2Fn7tTsAzvcvA\n4d9KZdntw7z6ou9z8Hnf5tUXfb/p0NR65n3yxl2CBcDNv9zMGf/ys3Zmc9yaNVeNR1+DGkGj9Fp/\nu3ztbjWUbTuCv13ulX3KwDUMK41ltw+z8Ot3sm0k+cAa3rKVhV9v/Ndt7eS0Wfv28fCTz9R979og\nUlZbtzdYiqRBeq0tDWoijdKtu7iGYaXxd99auzNYVGwbCf7uW7v/dVuZnDa8ZStBEly6KSgopyap\naFBzaZRuk4trGFYa9ZakaJReb3JaK520bHdeH+BTVL+5q1WfyQXL1nDNqgeavq91v1xrGJJOlrRO\n0npJ59U5f0a6PesaST+VdHjVufvT9DskDeWZT5t8xrIYXm2N5Pyla0bdR9IuAznNbehp8MHeKB1a\nBwvIr8/FJlZuNQxJPcBlwIkke3LfKml5RNxdddmvgOMi4jFJpwCLgXlV50+IiEfyyqNNXgdO6xv1\nTOpOWrZ74UlzuGDZGpbc8iAjEfRInD5vBp+Yf+i43rfRcN1mw3iX3PJgy/fNK8DZxMqzhnEUsD4i\n7ouIZ4BrgVOrL4iIn0bEY+nhKpKtWM1yV2+S2VgUtWz314Ye4JpVDzCStk2NRHDNqge4YNma3L5n\no1FnIxnaxzx5rxzyDBgDQPWfHhvStEbeB1xfdRzAdyWtlrSg0U2SFkgakjS0adOmcWXYJo/KJLOB\naX2I1n8Bv2CMy2XkpVEHfb2mofEONa4ouhnOitcRnd6STiAJGMdWJR8bEcOS9gdulHRvRNxUe29E\nLCZpymJwcNAtpSXW7k7n+XMHdrl/1nnfbnjt1DqN+JVlL7I2DRXRad5qqfLaPLWyddsIH/nqnTuP\ns66o6x33yiHPgDEMzKg6PihN24Wkw4ArgFMi4tFKekQMp/9ulHQdSRPXbgHDJodltw/zka/dyciO\nZ+dYfKTNM4in9fU2nC9Qb37G857Tw9CvN+/yV32laQjYJWgsu32Yc79yx879uYe3bOXcr9yxW/6r\nP8DbodVS5Qu/dufOiXZZ+3RGIjgnzXtW3nGvHPJskroVmC3pYElTgdOA5dUXSJoJLAXeGRH/WZW+\nt6R9Kq+B1wO/yDGv1uE+et2ancGiYmRH8NHrnm2z33tq4z6JLM0oo12G++Enn2k4Oqi2I/j8pXdR\n22+8I02vzmP1SKyxqi5royAwvGVr3VnZefGKteWQW8CIiO3A2cBK4B7gqxGxVtJZks5KL/sYsC9w\nec3w2enATyTdCfwc+HZE3JBXXq3zPfVM/TkT1em9PY1/nLM0nbTzQ622I3hrg2FG1eljmRtSz8Kv\n37kzaPQ0mOHXI03o7Gt3epdDrn0YEbECWFGTtqjq9fuB99e57z7g8Np0s2aa1RCyNIksPGnOLu39\nkPRTtONDPIt2NdtUZrfPnzvQcARTlpFN7eT+i3Lw0iDWFRrNG6tOb1ZDyFJ7qDdy6lNvHt+8htFo\nZw2nMru90eivgWl9DUd+mTXigGFd4YyjZ7ZMX3jSHHrrrEGRZfOfivlzB7j5vNfyq4vewM3nvbZt\nfxlnWQW2XXNDqjXb1OjCN72C3mZTuM1qOGBYV/jE/EM58+iZO9vkeyTOPHrmLiOR5s8d4OK3Hb7L\nh/AUwdtfNWNcH/yvfskLx33Pp9582G6/bFPS9IrqGk67NKo1VYYUX/zWw3eea6avd8rOfDXqF2mk\n2WAE6y6KEi1DOTg4GENDXnZqMquddwDJX9SVD8mxqrepUrPVbe+/6A1185Z1HkazOSFZ1ctDM82+\np4BfVb1flvWjKj739iPch9HhJK2OiMFW13XExD2zLLJ84DabdzCeD616O+695PwVdTuPG/0FXjtR\nsJvU9q/84N7sqyp0a5ltd26Ssq5Qb/+KestUNBpplMfEsU4ZgTQRavuARrNwYztqS9YZHDCsK7Sa\nsVzRaKRRHhPHmu3xULb1lsZbS3DQKAc3SVlXyFpzaDSXYrwTx+r1YTSbJF29XpNZWbiGYV0ha82h\n2aigsaoNFtB6j+96tR+zbucahnWF0dQc2t25PNa9vr3gnpWNA4Z1hUoA6JQ9tSt6p6jhAn5FLrg3\nfZ+phX1vKy8HDOt4leG01SNzfvv47xn69eamAaPRMNza/Ste3L8X9216ekyjmxoFC5GMJKru7J29\n/97ceO7xTd+vkufxuuWjJ7a8ZrR7YVSbvs/Uuku+W7k5YFhHqzcRDxrvO9Hovsow3K8NPbBLE9NI\nBP+18am257teGPmvjU9x4md+2DBoNCprHur9/4yGg8XklOtMb0knA58HeoArIuKimvNKz/858DTw\nnoi4Lcu99Ximdzmc+Jkf5vIhbs8amNY36iAxXqOdeW4TJ+tM79xGSUnqAS4DTgEOAU6XdEjNZacA\ns9OvBcAXR3GvlZCDxcSY6GBh5ZDnsNqjgPURcV9EPANcC5xac82pwNWRWAVMk3RAxnuthBwszDpX\nngFjAKjep3JDmpblmiz3mpnZBOr6iXuSFkgakjS0aVP2BdHMzGx08gwYw8CMquOD0rQs12S5F4CI\nWBwRgxEx2N/fP+5MW7Fm77930VkwswbyDBi3ArMlHSxpKnAasLzmmuXAu5Q4Gng8Ih7KeK+V0I3n\nHu+gMQE8YsnGIrd5GBGxXdLZwEqSobFXRsRaSWel5xcBK0iG1K4nGVb73mb35pVX6yytJrdNFqNZ\n4XUsAaD2nkbfr9l715tU2a78WefxjntmZpNc4fMwzMysXBwwzMwsEwcMMzPLxAHDzMwyccAwM7NM\nSjVKStKTQJn3xdwPeKToTOTI5eteZS4blL98cyJin1YXlW0/jHVZhoZ1K0lDLl/3KnP5ylw2mBzl\ny3Kdm6TMzCwTBwwzM8ukbAFjcdEZyJnL193KXL4ylw1cPqBknd5mZpafstUwzMwsJw4YZmaWSVcF\nDEkHSFLR+ciLpOlF5yFPfn7dy8+uu7Xr+XVFwJC0p6QvAj8CFkt6c9F5aidJz5X0OeB6Sf9cwvL5\n+XUpP7vu1u7n1xUBA/gL4ICIeBnwH8DHJb2s4Dy1haQB4N8BkWwm9SPgHwvNVPv5+XUvP7vu1tbn\n17EBQ9Jzqw4D2AQQEd8EbgDOkvSCIvLWZr8HroiIv46I3wJfBe6QdFjB+RoXP7/ufX5+dt377CDf\n59dxAUPSSyV9FbhK0hsk7Q1sBZ6QdGB62cXAXOCQ9J6uaVuVNEfSIkl9ABHxKPDDqktmAC+mS9fE\n8vPr3ufnZ9e9zw4m5vl1VMCQNAX4HPALkqriG4G/Ab4L/DFwuKQ9I+JhkurjhwGiSyaTSDqWpFwL\ngHPTNEXEU1WXTQXuj4g/FJDFcfHzA7r0+fnZAV367GDinl9HBQzgAGAL8Mm0+vQJ4LXAEcD/A94G\nHJ5eey3wmKTeIjI6Ro8CfwW8DHivpBfVeWBzgV8CSPpAl1WP/fy69/n52XXvs4MJen4dFTAiYhgY\nBE6sOr4c+HhEfB34L+B8SR8hKfR9EbGtqPyOVkTcA6yPiPXAjcDHYedfBxWvA/aV9A3gHSTtrF3B\nzw/ooudX3RxRxmdXU75SPTvYNe8T9vwiYsK/SNaWn151LGCP9PV7gJ9UnZtG0hn1qvS6/wZ8Hjiz\niLyPtXy1r4F9gPXA62ruvR5YC7y16HI0Kd8BwDE1aT0len67la8szy8t2/tK/uze1+BcVz+7NI8D\nwD8BU4t4fkUU+ALgXuDrwEVp2pSq81OA7wPnVKX9G/DKoh9WO8pX84DPAf4jfX16+lCPL7oMGcq4\nlmS0xZHpcXVA7Orn16p83fz8SNq0VwPnNjjf1c+uVfm6+dml+TyLpI/iUuC5dT5Xcn9+E1nYPYGL\n0ojXD0wHngZemJ6v/tD5E+BXwHzgzPSH4JCiH9h4yldzbXVZHwMeB/4VeE7R5WhRxinpD+r1wP8l\n6Tjbu3Kum59fhvLtVkvspudH8lfpk0BvnXNd/bvXqnxNytotz65SM7qk0Yd/1TW5Pr+J3HFvG/D5\niHgIQNLxwDdJqpCbo1JqaUpErJZ0DnAkcALw1xFx9wTmdSyalq/6wogISc8nqY08CHwwIm6e2OyO\nXkTsAH4n6ZckY7tnkfyA3pSeQ1JPlz6/VuWLquu68fldARwL7Cnp4PT13RGxqgS/e9CkfNUXdeOz\nS/Ms4HjgE5JeQdL8tBZYHhGb02vy/93LMSruS7LG+nF1ouDrgfuBTwM/J6kS9lRf0+lfYy1f1bVT\ngMOKLsdoypemzwKuSl+fS/KX+DnAQd3+/LKUrxueX5Oy/Q3JHzY/Ay4Efg28qwy/e1nK1w3PrlH5\ngB6SjvrPAt8B/gewAvgH4I8r5co9bzkV+EXAdSRD2RZR0yyT/odMSV+/AbiVFlXJTvpy+fhC+gP8\nzyQjSZYUnWeXr3XZgBeS/GW9V3r8JmBV7QdqJ39N1vKR9LF8gGQOxQfStEOBK4HXTFT+8hpW+zvg\nk8DB6X/A62qGgD0aaRNGRHw7vX4gp7zkYdKWLx2bfihwF0lz2z8B6yXNLiivY1Hm8jUsWyRNF5+I\niKfT42+RzAQ+qKjMjsFkLN8ekUSI75EMjz0SICLWkIzI3DFRmcslYEQy5X5tRDwBLCFpktn5gVn1\ny/nydMzzr4Hf5JGXPEzm8kXEXcBPgA9HxF+QdBg+TYePWa9W5vK1+tmskHSIpKUkTael+NmsVrLy\nHZieuw+4DHiBpM9K+ibJZ/ivJjKD46k+9ZFhdAHJENMP82wzTR/wTpK/4j5UdDXQ5ctcvql558nl\ny61slZ/NXuAvS/izWfby7VGVti/wFhrMN8k13+Mo8HRgA3BJTXr18MpKZ9pxwHLgpcD7gD8iqUr1\nFf3gXL5Rl++vgBcVnf/JXL5x/mweQDKhq5OHkbp8jX82Dy4y7+NpkppCMob5tZJeDjuH5VWGVx4W\nESMAEfEjkvHttwFnpP8Zj0TE1nF8/7y5fI3LN1JMlkelzOUbT9mmRMSWiOjkJjaXr3753kEyCqw4\no4iK1VWiKSRLAb+fpIPmWzXR82qSduADSSa0vYekHX9+0dHd5XP5uq18ZS6by9dd5ctUWJKRIp8H\n/qwq/Wj6U1deAAADCUlEQVTgK+nrO0lmFh4GvIaatkNgZtEFdflcvm4rX5nL5vJ1Z/maNkmlswu/\nQNIm/3Pgf0v6kKQ9SWbCVmZR3gksBRZExE0RcVl6/x4AEfFAs+9TFJfP5YPOLF+ZywYuH11avlZL\ng+xDsp76SRHxpKRHSCaivYFkEaxzJc0nafMdAtbAzo1JIiK255f1tnD5XL5OVeaygcvXleVrWsOI\nZCzw/STtaAA3k3S+nAhsJ9lU/N8j4rUksxDfLWmvSOtSnc7lc/k6VZnLBi4fXVq+LIsPXgecLOmA\niHhI0hqS9rZpEfFB2BkV7wSOyTGveXH5XL5OVeaygcvXdeXLMqz2J8AjpJEyIlYDRwHPgZ2rk3Z0\nVGzB5XP5OlWZywYuX9eVr2XAiGS57m8Cp0h6m6RZJMskbE/Pd/qY9aZcPpevU5W5bODydWP5lDXA\nSTqFZCPxY4BLI+LSPDM20Vy+7lbm8pW5bODydZPMAQNAUi/Jfh4d2YM/Xi5fdytz+cpcNnD5usWo\nAoaZmU1eee2HYWZmJeOAYWZmmThgmJlZJg4YZmaWiQOGmZllkmVpEDNrQdIIyQJyvSQTs64GPhvp\npjhmZeCAYdYeWyPiCABJ+wNfBp4HXFhorszayE1SZm0WERuBBcDZSsyS9GNJt6VfxwBIujpd4pr0\n+EuSTi0q32ateOKeWRtI+l1EPLcmbQswB3gS2BERv5c0G1gSEYOSjgM+HBHzJT0fuAOY3e2zga28\n3CRllr9e4FJJR5BsmPMygIj4kaTLJfUDbwG+4WBhncwBwywHkl5MEhw2kvRjPAwcTtIM/PuqS68G\nzgROA947wdk0GxUHDLM2S2sMi0hWJo20uWlDROyQ9G6gp+ryq0j2fP5tRNw98bk1y84Bw6w9+iTd\nwbPDav8d+Ex67nLgG5LeBdwAPFW5KSIelnQPsGyC82s2au70NiuQpL1I5m8cGRGPF50fs2Y8rNas\nIJL+DLgHuMTBwrqBaxhmZpaJaxhmZpaJA4aZmWXigGFmZpk4YJiZWSYOGGZmlsn/B3owpO+YGiva\nAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x119ea8048>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gracc_jobs = []\n",
    "for hit in missing_jobs:\n",
    "    gracc_jobs.append([dateutil.parser.parse(hit['EndTime']), hit['CoreHours']])\n",
    "\n",
    "sge_jobs = np.array(sge_jobs)\n",
    "gracc_jobs = np.array(gracc_jobs)\n",
    "#plt.scatter(sge_jobs[:, 0].astype(\"datetime64[ns]\"), sge_jobs[:, 1])\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "\n",
    "#gracc_plot = ax.scatter(gracc_jobs[:, 0], gracc_jobs[:, 1], label=\"GRACC\")\n",
    "sge_plot = ax.scatter(sge_jobs[:, 0], sge_jobs[:, 1], label=\"SGE\")\n",
    "#ax.legend(handles=[gracc_plot, sge_plot])\n",
    "fig.autofmt_xdate()\n",
    "ax.set_xlim([datetime.date(2019, 4, 13), datetime.date(2019, 4, 16)])\n",
    "plt.ylabel(\"Core Hours\")\n",
    "plt.xlabel(\"Day\")\n",
    "\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}