Get a filtered Hostlist and Tags from DataDog

DataDogExtract.py

#!/usr/bin/env python3

import json
import requests
import os,sys,time,getopt
import configparser

helpinfo="""
Usage:
-s IncludeList
-x ExcludeList
-r ReportList
-a PrintAllTags

example: %s -s type:web,site:us -x webserver:iis,os:windows -a

Config file %s looks like:

[auth]
api_key=blah
application_key=blahblah

[file]
cachefile=/tmp/overview.json
cachefileage=28800
""" % (sys.argv[0], os.path.expanduser('~/.DataDog.cfg'))

try:
    args = getopt.getopt(sys.argv[1:],"ar:s:x:")
    alltags=False
    includelist=False
    excludelist=False
    reportlist=False
    for arg in args[0]:
        switch, data = arg
        if switch == "-s":
            includelist=data.split(",")
        if switch == "-x":
            excludelist=data.split(",")
        if switch == "-r":
            reportlist=data.split(",")

        if switch == "-a":
            alltags = True

except:
    print(helpinfo)
    exit(1)

config = configparser.ConfigParser()
config.read(os.path.expanduser('~/.DataDog.cfg'))

cachefile = config.get("file","cachefile")
cachefileage = float(config.get("file","cachefileage"))

def SearchIndex(searchlist, excludelist):
    matchindex={}
    hostlist=[]
    if includelist:
        for searchtag in searchlist:
            if searchtag in tagindex:
                for host in tagindex[searchtag]:
                    if host in matchindex:
                        matchindex[host]+=1
                    else:
                        matchindex[host]=1
            else:
                print('Unknown Tag:',searchtag)
                exit()
    else:
        hostlist=hostindex.keys()

    if excludelist:
        for searchtag in excludelist:
            if searchtag in tagindex:
                for host in tagindex[searchtag]:
                    if host in matchindex:
                        matchindex[host]-=1
                    else:
                        matchindex[host]=0

    for host, count in matchindex.items():
        if count == len(searchlist):
            hostlist.append(host)

    return(hostlist)

def gettags(hostname):
    alltags=""
    for tag in hostindex[hostname]:
        if alltags != "":
            alltags += "," + tag
        else:
            alltags += tag
    return(alltags)

def selecttags(host,taglist):
    matched=""
    for tag in hostindex[host]:
        k=tag.split(":",1)
        for x in taglist:
            if k[0] == x:
                if k[1]=="":
                    k[1]="None"
                if matched != "":
                    matched += "\t" + k[0]+":"+k[1]
                else:
                    matched += k[0]+":"+k[1]
    return(matched)

def CreateIndex():
    hostindex={}
    tagindex={}
    for node in data["rows"]:

        if len(node["tags_by_source"])==0:
            node["tags_by_source"]["EmptySet"]=["NULL:NULL"]
            
        for source in node["tags_by_source"]:
            for tag in node["tags_by_source"][source]:
             
                if node["display_name"] not in hostindex:
                    hostindex[node["display_name"]] = []

                if tag not in tagindex:
                    tagindex[tag] = []
                
                tagindex[tag].append(node["display_name"])
                hostindex[node["display_name"]].append(tag)
    return(hostindex,tagindex)

try:
    if (time.time() - os.stat(cachefile).st_mtime) > cachefileage:
        raise StaleCacheFile("File Exceeds Age")

    f = open(cachefile,"r")
    data=json.load(f)
    f.close()

except:
    sys.stderr.write("Refreshing Cache from Web: "+cachefile+"\n")
    params = {
      'api_key': config.get("auth","api_key"),
      'application_key': config.get("auth","application_key")
    }

    response = requests.get('https://app.datadoghq.com/reports/v2/overview', params=params)
    data = json.loads(response.content)

    os.umask(0o177)
    f = open(cachefile,"w")
    json.dump(data,f)
    f.close()

hostindex,tagindex = CreateIndex()

if len(sys.argv) == 1:
    hostlist = hostindex.keys()
else:
    hostlist = SearchIndex(includelist,excludelist)

for host in hostlist:
    if alltags:
        print(host,gettags(host),sep='\t')
    elif reportlist != False:
        print(host,selecttags(host,reportlist),sep='\t')
    else:
        print(host)