download_all_crash_ids_bug_1270211.py

import time
import requests
from urllib import urlencode

URL = "https://crash-stats.mozilla.com/api/SuperSearch/"
params = {
    'ipc_channel_error': '!__null__',
    'date': '>=2016-04-27',
    '_columns': ['uuid'],
}


crash_ids = []
total = None
def download(auth_token, offset=0):
    p = dict(
        params,
        _results_offset=offset,
        _results_number=500,
    )
    url = URL + '?' + urlencode(p, True)
    print url
    results = requests.get(url, headers={
        'Auth-Token': auth_token
    }).json()
    global total
    if total is None:
        total = results['total']
        print "TOTAL", total
    crash_ids.extend([
        x['uuid'] for x in results['hits']
    ])
    print "\t#crashes", len(crash_ids)
    print "\toffset", offset
    if offset < total:
        time.sleep(2)  # be nice
        download(auth_token, offset + 500)


import sys
download(sys.argv[1])
print len(crash_ids)
print "#unique", len(set(crash_ids))

# import json
# with open('crash_ids_1270211.json', 'w') as f:
#     json.dump({'crash_ids': list(set(crash_ids))}, f, indent=2)

sql = "INSERT INTO reprocessing_jobs (crash_id) VALUES "
sql += ',\n'.join("('%s')"%x for x in list(set(crash_ids)))
sql += ';\n'
open('crash_ids_1270211.sql', 'w').write(sql)