ThreatStream API Intelligence downloader, separates intel into different files based on i-type.

threatstream_intelligence_walker.py

#!/usr/bin/env python

"""ThreatStream Raw Intelligence downloader.  Puts into 1 file for each itype."""

__author__ = 'Jay Taylor [@jtaylor]'

import io, json, requests, os

################################################################################
# Begin configuration

username = 'YOUR_TS_ACCOUNT_EMAIL_ADDRESS'
api_key = 'YOUR_TS_ACCOUNT_API_KEY'

local_prefix_path = 'indicators'

base_url = 'https://api.threatstream.com'

# End configuration
################################################################################


next_url = '{}/api/v1/intelligence/?username={}&api_key={}&limit=1000&order_by=-id'.format(base_url, username, api_key)

if not os.path.exists(local_prefix_path):
    os.mkdir(local_prefix_path)

itype_file_handles = {}
itype_intel_written = [] # Keep track of whether or not we've written intelligence to this stream (used to appropriately inject commas).

while next_url is not None:
    print 'fetching {}'.format(next_url)
    response = requests.get(next_url)

    data = response.json()

    next_url = '{}{}'.format(base_url, data['meta']['next'])

    for intelligence in data['objects']:

        itype = intelligence['itype']

        if itype not in itype_file_handles:
            itype_file_handles[itype] = io.open('{}/{}.json'.format(local_prefix_path, itype), 'w+b')
            itype_file_handles[itype].write('[')

        if itype in itype_intel_written:
            itype_file_handles[itype].write(',')
        else:
            itype_intel_written.append(itype)

        itype_file_handles[itype].write(json.dumps(intelligence))

# Close all open files.
map(lambda fh: fh.write(']') and fh.close(), itype_file_handles.values())

print 'all done'