Retrieve scope from HackerOne (using their directory) + all public reports (commented part)

retrieve_scope.py

import requests
import csv
import json

hackerone_url = "https://hackerone.com"
page = 1
session = requests.Session()
# titles = []

reports = []
tmp_url = "{}/directory".format(hackerone_url)
# print tmp_url
req = session.get(tmp_url)

tmp_url = "{}/programs/search?query=type%3Ahackerone&sort=published_at%3Adescending&page={}".format(hackerone_url, page)
data = session.get(tmp_url).json()

total_programs = data['total']
# print "Total number of programs: {}".format(total_programs)
programs = []
while ((page - 1) * 100) <= total_programs:
    tmp_url = "{}/programs/search?query=type%3Ahackerone&sort=published_at%3Adescending&page={}".format(hackerone_url, page)
    data = session.get(tmp_url).json()
    for entry in data['results']:
        handle = entry['handle']
        # print('Found program {}'.format(handle))
        # print entry['meta']
        # bug_count = entry['meta'].get('bug_count')
        # minimum_bounty = entry['meta'].get('minimum_bounty')
        # default_currency = entry['meta'].get('default_currency')
        program = {
            'handle': handle.encode('utf-8'),
            # 'minimum_bounty': minimum_bounty,
            # 'default_currency': default_currency,
            # 'bug_count': bug_count
        }
        programs.append(program)
    page = page + 1

for program in programs:
    page = 1
    session = requests.Session()
    titles = []

    # print "{} - Bug count: {} - Min payout: {} {}".format(program['handle'], program['bug_count'], program['minimum_bounty'], program['default_currency'])
    company_name = program['handle']
    tmp_url = "{}/{}".format(hackerone_url, company_name)
    # print tmp_url
    headers = {'Accept': 'application/json'}
    data = session.get(tmp_url, headers=headers).json()
    program['scopes'] = data['scopes']
    # print program
    # print '\n'
    # programs[program]['scopes'] = data['scopes']
    # tmp_url = "{}/{}/activities?page={}".format(hackerone_url, company_name, page)
    # print tmp_url
    # data = session.get(tmp_url).json()

    # total_pages = data['total_pages']
    # print "Total number of pages: {}".format(total_pages)
    # while page <= total_pages:
    #     tmp_url = "{}/{}/activities?page={}".format(hackerone_url, company_name, page)
    #     data = session.get(tmp_url).json()
    #     for entry in data['activities']:
    #         if entry.get('public_report') is not None:
    #             title = entry['public_report']['title'].encode('utf-8')
    #             url = entry['public_report']['url'].encode('utf-8')
    #             bounty_amount = entry.get('bounty_amount', 'N/A')
    #             bounty_currency = entry.get('bounty_currency', 'N/A')
    #             updated_at = entry.get('updated_at', 'N/A')
    #             try:
    #                 username = entry['reporter']['username'].encode('utf-8')
    #             except:
    #                 username = 'N/A'
    #             if title not in titles:
    #                 titles.append(title)
    #                 print "{} ({}{}) by {}".format(title, hackerone_url, url, username)
    #                 tmp = {
    #                     'title': title,
    #                     'url': url,
    #                     'bounty_amount': bounty_amount,
    #                     'bounty_currency': bounty_currency,
    #                     'username': username,
    #                     'updated_at': updated_at
    #                 }
    #                 reports.append(tmp)
    #     page = page + 1

print json.dumps(programs)

# keys = reports[0].keys()
# with open('results.csv', 'wb') as output_file:
#     dict_writer = csv.DictWriter(output_file, keys)
#     dict_writer.writeheader()
#     dict_writer.writerows(reports)