Experimental_traceback analysis

get_data.sh

#First, you want to enable `experimental_traceback` in Django Admin's Experimental Features.
#Let some data collect in influx. 

wget -O data.json https://influxdb-dramafever.drama9.com:8443/query?q=select+*+from+short_term.experimental_traceback%3B&db=appmetrics

#now we have a big JSON object containing 10,000 rows of experimental_traceback data from Influx.

python processIndivLines.py | sort -r -n

#This finds the interesting lines from the database, counts how often they're used, and spits out sorted csv output.

processIndivLines.py

import json

## If you look at the tracebacks, they're typically organized like so:
## gunicorn stuff | dramafever stuff | django stuff
## databaseLines is a list of the first "django stuff", as it were. We're
## going to split up the tracebacks and look for the last piece of code before
## we enter the django stuff. 
databaseLines = ["o/db/models/query.py_328",
        "db/models/manager.py_127",
        "ls/fields/related.py_614",
        "o/db/models/query.py_162",
        "o/db/models/query.py_586",
        "ngo/template/base.py_849",
        "basic/controllers.py_181",
        "ngo/template/base.py_789",
        "ngo/template/base.py_648",
        "plate/defaulttags.py_937",
        "plate/defaulttags.py_322",
        "plate/defaulttags.py_224",
        "ngo/template/base.py_919",
        "ngo/template/base.py_905",
        "ngo/template/base.py_202",
        "ngo/template/base.py_210",
        "/django/templates.py_37",
        "o/template/engine.py_221",
        "o/template/loader.py_115",
        "/django/shortcuts.py_45",
        "e/backends/django.py_74",
        "/utils/decorators.py_145",
        "plate/loader_tags.py_135",
        "pi/function_trace.py_98",
        "plate/loader_tags.py_159",
        "rib/auth/backends.py_93",
        ]

data = open("data.json", "r").read()

datamap = json.loads(data)

datavalues = datamap["results"][0]["series"][0]["values"]

tracebackcountmap = {}

unknowntracebacks = set()

for dv in datavalues:
    codeLines = dv[4].split("|")
    added = False
    for i in range(0, len(codeLines)):
        if codeLines[i] in databaseLines:
            if codeLines[i-1] not in tracebackcountmap:
                tracebackcountmap[codeLines[i-1]] = 0
            tracebackcountmap[codeLines[i-1]] = tracebackcountmap[codeLines[i-1]] + 1
            added = True
            break
    if not added:
        unknowntracebacks.add(dv[4])

outstring = ""
for key, val in tracebackcountmap.iteritems():
    outstring = outstring + "{}, {}\n".format(val, key)

print outstring

###### If the above fails to process some tracebacks, you can print out the failed tracebacks and adjust `databaseLines`

#print "\n"*4 + "="*80 + "\n"*4

#print "Found {} unknown tracebacks".format(len(unknowntracebacks))

#print ("\n"*2).join(unknowntracebacks)