Dump Crowdstrike Falcon host data into elasticsearch

falcondump.py

import requests
import json
import sys
import time
import datetime
from requests.auth import HTTPBasicAuth
import logging 
import elasticsearch
import geoip
import traceback

g=geoip.open_database('/etc/geoip.mmdb')
logging.basicConfig(format='FalconHostDump: %(asctime)-15s  %(message)s')
LOG = logging.getLogger('FalconHostDump')
LOG.setLevel(logging.DEBUG)

class FalconAuth:
    def __init__(self,client_id,client_secret):
        self.client_id=client_id
        self.client_secret=client_secret
    def newtoken(self):
        response=requests.post("https://api.crowdstrike.com/oauth2/token",data={"client_id":self.client_id,"client_secret":self.client_secret},
            headers={"Content-Type":"application/x-www-form-urlencoded","Accept":"application/json"})
        if not response.status_code==201:
            return None
        json_data=response.json()
        json_data["expires"]=time.time()+json_data["expires_in"]
        return json_data

    def getToken(self):
        tokendata=''
        with open("OAuth2.json","r") as f:
            try:
                tokendata=json.loads(f.read())
                if tokendata['expires'] < time.time():
                    tokendata=self.newtoken()
            except Exception:
                print("Error loading oauth2 data")
                traceback.print_exc()
                tokendata= self.newtoken()
        with open("OAuth2.json","w+") as f:
            f.write(json.dumps(tokendata))
            return tokendata['access_token']
                
def geoip(r):
    r['geoip']="0.0,0.0"#
    r['location']={"lat":0.0,"lon":0.0}

    try:
        match=g.lookup(r['external_ip'])
        cn=match.country
        if not cn:
            cn="N/A"
        r["Country"]=cn

        if match and len(match.location)==2:
            r['location']['lon']=r['lon']=match.location[0]
            r['location']['lat']=r['lat']=match.location[1]
            r['geoip']=str(match.location[0])+","+str(match.location[1])
        else:
            r['location']['lon']=r['location']['lat']=r['lon']=r['lat']=0.0
            r['geoip']="0.0,0.0"
    except:
        LOG.exception('geoip')
        r['location']['lon']=r['location']['lat']=r['lon']=r['lat']=0.0
        r['geoip']="0.0,0.0"
        
class ES:

    def __init__(self, config, logger):
        import elasticsearch
        self.es = elasticsearch.Elasticsearch(hosts=config['hosts'],http_auth=("esuser","espassword"))
        self.index_name = config['index']
        self.lh = logger
        self.doctype = config['doctype']

    def create(self, data, id):
        try:
            self.es.create(index=self.index_name,
                           doc_type=self.doctype, id=id, body=data)
        except elasticsearch.ConflictError:
            self.lh.info("Updating host "+str(id))
            self.es.delete(index=self.index_name,doc_type=self.doctype,id=id)
            self.es.create(index=self.index_name,doc_type=self.doctype, id=id, body=data)

        except Exception as e:
            self.lh.exception('Elasticsearch index error:' + str(e))
        self.lh.debug(json.dumps(data,sort_keys=True,indent=4))

def dumphosts(conf):
    global LOG
    es = ES(conf,LOG)
    query_api_user=conf["query_api_user"]
    query_api_password=conf["query_api_password"]
    tstamp =  datetime.datetime.fromtimestamp(time.time()-(86400*7)).strftime('%Y-%m-%d')#2016-07-19T11:14:15Z
    auth = FalconAuth("apiuser","apisecret")
    resp=requests.get("https://api.crowdstrike.com/devices/queries/devices/v1?limit=5000&last_seen='>"+tstamp+"'",headers={"Authorization":"Bearer "+auth.getToken()})
    
    total=0
    if resp.status_code==200:
        data=json.loads(resp.text)
        total=data['meta']['pagination']['total']
    if total<1:
        LOG.error("unable to get host total")
        return
    aid_list=[]
    LOG.info("About to dump "+str(total)+" Hosts' Data.")
    for offset in range(0,total,5000):
        resp=requests.get("https://api.crowdstrike.com/devices/queries/devices/v1?filter=platform_name:'Windows'&limit=5000&offset="+str(offset).strip(),headers={"Authorization":"Bearer "+auth.getToken()})
        if resp.status_code == 200:
            data=json.loads(resp.text)
            for resource in data["resources"]:
                aid_list.append(resource)
    aid_groups={}
    index=0
    i=0         
    for aid in aid_list:
        if not i in aid_groups:
            aid_groups[i]=[]
        aid_groups[i].append(aid)
        if index>=120:
            index=0
            i+=1
        index+=1
    out="AID,Hostname,Last seen,Product type,Prevention Policy,OS Version,Build Number\n"
    kvlist=[]
    sofar=0
    counter=0
    for grp in aid_groups:
        time.sleep(5)
        aids="&ids=".join(aid_groups[grp]).strip("&ids=").strip()
        sofar+=len(aid_groups[grp])
        LOG.info("Dumped:"+str(sofar)+"/"+str(total)+" Hosts")
        resp=requests.get("https://api.crowdstrike.com/devices/entities/devices/v1?ids="+aids,headers={"Authorization":"Bearer "+auth.getToken()})
    
        if resp.status_code == 200 or resp.status_code == 404:
            if resp.status_code == 404:
                resp=requests.get("https://api.crowdstrike.com/devices/entities/devices/v1?ids="+aids,headers={"Authorization":"Bearer "+auth.getToken()})
            jdata=json.loads(resp.text) 
            if "resources" in jdata:
                for r in jdata["resources"]:
                    counter+=1
                    geoip(r)
                    es.create(r,r['device_id'])
    LOG.info("Finished host dump")
    
def main():
    while True:
        conf={}
        conf["query_api_user"]="replaceme"
        conf["query_api_password"]="replaceme"
        conf["hosts"]=["http://esuser:espassword@10.0.0.1:9200"]
        conf["index"]="falconhosts"
        conf["doctype"]="hostdata"
        dumphosts(conf) 
        time.sleep(86400)
    
if __name__ == '__main__':
    global LOG
    try:
        main()
    except Exception as e:
        LOG.exception("main() exception")
        sys.exit(1)