Rapid7-OpenAPI-DNS Scan extractor

r7-data-scraper.py

import requests
import json
from pprint import pprint

'''
Author: @awhitehatter
Date: 01/04/2019
Version: 1.0
Description: A script that pulls down latest FDNS and RDNS scans from rapid7, provides the URL

**NOTE** This requires an API Key w/Rapid7 
'''

# Hardcoded Credz, super secret sekure brah
apikey = ''

# Some other useful variables
url = 'https://us.api.insight.rapid7.com/opendata/studies/'
headers = {"X-Api-Key": apikey}
rdns_url = url + 'sonar.rdns_v2/'
fdns_url = url + 'sonar.fdns_v2/'

# Grab the latest RDNS data set
rdns_data = requests.get(rdns_url, headers=headers)
rdns_json = json.loads(rdns_data.text)
rdns_latest_scan = rdns_json['sonarfile_set'][0]
rdns_url = rdns_url + rdns_latest_scan + '/download/'
r = requests.get(rdns_url, headers=headers)
r_json = json.loads(r.text)
rdns_dl = r_json['url']

# Grab the latest FDNS data sets
# They've changed how they store this data set, so I'm trying to target just A, AAAA, CNAME, ANY, and MX records
fdns_data = requests.get(fdns_url, headers=headers)
fdns_json = json.loads(fdns_data.text)

# Build some empty lists to collect data sets
a_sets = []
aaaa_sets = []
cname_sets = []
any_sets = []
mx_sets = []

for i in fdns_json['sonarfile_set']:
    if "fdns_a.json.gz" in i:
        a_sets.append(i)
    elif "fdns_aaaa.json.gz" in i:
        aaaa_sets.append(i)
    elif "fdns_cname.json.gz" in i:
        cname_sets.append(i)
    elif "fdns_mx.json.gz" in i:
        mx_sets.append(i) 
    elif "fdns_any.json.gz" in i:
        any_sets.append(i)

# Finally, pull the latest scan 
# Grab DL link for A records
fdns_a_latest = a_sets[0]
fdns_a_url = fdns_url + fdns_a_latest + '/download/'
r = requests.get(fdns_a_url, headers=headers)
r_json = json.loads(r.text)
fdns_a_dl = r_json['url']

# Grab DL link for AAAA records
fdns_aaaa_latest = aaaa_sets[0]
fdns_aaaa_url = fdns_url + fdns_aaaa_latest + '/download/'
r = requests.get(fdns_aaaa_url, headers=headers)
r_json = json.loads(r.text)
fdns_aaaa_dl = r_json['url']

# Grab DL link for CNAME records
fdns_cname_latest = cname_sets[0]
fdns_cname_url = fdns_url + fdns_cname_latest + '/download/'
r = requests.get(fdns_cname_url, headers=headers)
r_json = json.loads(r.text)
fdns_cname_dl = r_json['url']

# Grab DL link for MX records
fdns_mx_latest = mx_sets[0]
fdns_mx_url = fdns_url + fdns_mx_latest + '/download/'
r = requests.get(fdns_mx_url, headers=headers)
r_json = json.loads(r.text)
fdns_mx_dl = r_json['url']

# Grab DL link for ANY records
fdns_any_latest = any_sets[0]
fdns_any_url = fdns_url + fdns_any_latest + '/download/'
r = requests.get(fdns_any_url, headers=headers)
r_json = json.loads(r.text)
fdns_any_dl = r_json['url']

print("rdns: " + rdns_dl)
print("fdns_a: " + fdns_a_dl)
print("fdns_aaaa" + fdns_aaaa_dl)
print("fdns_cname: " + fdns_cname_dl)
print("fdns_mx: " + fdns_mx_dl)
print("fdns_any_dl: " + fdns_any_dl)