wget json gadget for splunk

wget.py

#!/usr/bin/env python
# supports_getinfo allows us to specify all but maxinputs and filename at parse time
# [wget]
# supports_getinfo = True
# maxinputs = 0
# filename = wget.py

import sys, time, json
from collections import OrderedDict
import splunk.Intersplunk as si

def error(*e):
    e = ' '.join([ str(i) for i in e ])
    si.outputResults(si.generateErrorResults(e))
    sys.exit(0)

def _explode_json(event, dat, base='', keep=True):
    ret = list()
    if keep:
        ret.append(event)
    if isinstance(dat, dict):
        for k,v in dat.iteritems():
            if isinstance(v, dict):
                ret.extend(_explode_json(event, v, base=k, keep=False))
            elif isinstance(v, list):
                for i in v:
                    new_event = event.copy()
                    new_event['_raw'] = json.dumps(i)
                    ret.extend(_explode_json(new_event, i, base=k, keep=True))
            else:
                event['{0}.{1}'.format(base,k) if base else k] = dat[k]
    return ret

def explode_json(event):
    try:
        ret = _explode_json(event, json.loads(event['_raw']))
        for item in ret[1:]:
            for k in ret[0]:
                if k not in item:
                    item[k] = ret[0][k]
        return ret
    except Exception as e:
        error('ERROR exploding json:', e)
        
def get_doc(url, stype, eventkeys):
    try:
        import requests
        if not url.startswith('http://') and not url.startswith('https://'):
            url = 'http://' + url

        try:
            res = requests.get(url)
        except Exception as e:
            error("ERROR during GET: ", e)
            return

        ct = res.headers.get('content-type', 'text/plain')

        event = OrderedDict()
        event['_raw']             = res.text
        event['_time']            = time.time()
        event['sourcetype']       = stype or 'wget:' + ct.split(';')[0]
        event['res.url']          = res.url
        event['res.ok']           = res.ok
        event['res.status_code']  = res.status_code
        event['res.reason']       = res.reason
        event['res.encoding']     = res.encoding
        event['res.content_type'] = ct

        if eventkeys:
            if 'json' in ct:
                return explode_json(event)
            else:
                error('eventkeys specified, but document seems not to be json:', ct)
        return (event,)
    except Exception as e:
        error('ERROR formatting event:', e)

def usage():
    si.parseError('''usage: wget [url=]http://whatever [stype=something] [explode[_json]]''')

    if __name__ == '__main__':
    try:
        # ../../search/bin/return.py
        # ../../search/bin/rangemap.py
        # ../../search/bin/trendline.py
        (isgetinfo, sys.argv) = si.isGetInfo(sys.argv)
        if isgetinfo:
            si.outputInfo(streaming=True, generating=False, retevs=True, reqsop=True,
                preop='', timeorder=False)

        args,kwargs = si.getKeywordsAndOptions()
        url         = kwargs.pop('url', None)
        stype       = kwargs.pop('stype', None)

        def kw(*kl):
            ret = False
            for k in kl:
                if k in args:
                    ret = True
                    args.remove(k)
            return ret
        eventkeys = kw('explode_json', 'explode')

        if url is None:
            if len(args) > 0:
                url = args.pop(0)
            else:
                usage()

        if args or kwargs:
            usage()

        # ../../search/bin/erex.py
        results = si.readResults(None, None, True)
        events = get_doc(url, stype, eventkeys)
        if events:
            results.extend(events)
        si.outputResults(results)

    except Exception as e:
        error('ERROR during setup:', e)