Skip to content

Instantly share code, notes, and snippets.

@peterbe peterbe/download.py

Last active Jan 31, 2016
Embed
What would you like to do?
Download from bugzilla.mozilla.org, Upload to autocompeter.com. See http://www.peterbe.com/plog/a-quicksearch-for-bugzilla-using-autocompeter for context.
import datetime
import json
import os
import hashlib
import time
import requests
def bulk_upload(auth_key, documents):
t0 = time.time()
print requests.post(
'https://autocompeter.com/v1/bulk',
data=json.dumps({'documents': documents}),
headers={
'Auth-Key': auth_key,
}
)
t1 = time.time()
print "Took %.2f seconds to upload %d documents" % (
t1 - t0,
len(documents)
)
def json_download(url, params):
key = '{}{}'.format(url, params)
key += datetime.datetime.utcnow().strftime('%d')
hash_key = hashlib.md5(key).hexdigest()
if not os.path.isdir('.downloadcache'):
os.mkdir('.downloadcache')
cache_file = '.downloadcache/{}.json'.format(hash_key)
try:
with open(cache_file) as f:
return json.load(f)
except IOError:
pass
response = requests.get(url, params=params)
if not response.status_code == 200:
print response.status_code
raise ValueError(response.content)
json_ = response.json()
with open(cache_file, 'w') as f:
json.dump(json_, f, indent=2)
return json_
def run(autocompeter_auth_key, products):
BUGZILLA_URL = 'https://bugzilla.mozilla.org/rest/'
config_url = BUGZILLA_URL + 'product'
params = {
'type': 'accessible',
'include_fields': 'name,components.name',
}
stuff = json_download(config_url, params=params)
for product in stuff['products']:
if product['name'] in products:
print "PRODUCT", product['name']
for component in product['components']:
url = BUGZILLA_URL + 'bug'
print "COMPONENT", component['name']
params = {
'include_fields': 'status,id,resolution,summary',
'product': product['name'],
'component': component['name'],
}
bugs = json_download(url, params=params)
print "\t", len(bugs['bugs']), "bugs"
documents = []
for bug in bugs['bugs']:
if bug['resolution']:
status = '{}, {}'.format(bug['status'], bug['resolution'])
else:
status = bug['status']
title = u'{}: {} ({})'.format(
bug['id'],
bug['summary'],
status,
)
documents.append({
'url': 'https://bugzilla.mozilla.org/show_bug.cgi?id={}'.format(bug['id']),
'title': title,
'popularity': bug['id'],
'group': product['name'],
})
bulk_upload(autocompeter_auth_key, documents)
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
'auth_key', type=str,
help="Get your own key on https://autocompeter.com"
)
parser.add_argument(
'products', nargs='+',
help="Each product you want to store"
)
args = parser.parse_args()
import sys
assert len(args.auth_key) == 24, "Autocompeter auth keys are always 24 characters"
sys.exit(run(args.auth_key, args.products))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.