Skip to content

Instantly share code, notes, and snippets.



Last active Feb 15, 2016
What would you like to do?
Github Issue Relation downloader
#!/usr/bin/env python
from __future__ import print_function
import os
import sys
import requests
import keyring
import getpass
import argparse
import re
import pickle
from collections import defaultdict
from graphviz import Digraph
parser = argparse.ArgumentParser(description="Parse issue comments and build a reference tree",
parser.add_argument('--new-token', action='store_true', help='Change Github API token')
parser.add_argument('owner', nargs=1, help='owner for github repo to inspect')
parser.add_argument('repo', nargs=1, help='repository under github owner to inspect')
parser.add_argument('--output', '-o', dest='output', nargs=1, default='-', help='Output destination')
parser.add_argument('--intermediate', dest='intermediate_datafile', default=None, help='If specified, save API data locally.')
parser.add_argument('--local-load', dest='load_intermediate', action="store_true", help='Do not contact API, use --intermediate pickle file.')
args = parser.parse_args()
github_token = keyring.get_password(UUID, USER)
if (github_token is None) or args.new_token:
new_token = getpass.getpass('Enter Github API Key (characters will not be shown):')
keyring.set_password(UUID, USER, new_token)
print("New token set.")
github_token = new_token
if github_token is None:
raise Exception("No Github API token set.")
next_url = "{}/{}/issues".format(args.owner[0], args.repo[0])
# Tree of discovered issues stored in target <- sources target form, deduped.
data = {
'issue_tree' : defaultdict(set),
'issue_nums' : dict()
rx = re.compile('''#\d+''')
def extract_crossrefs(body):
refs = []
m = rx.findall(body)
for s in m:
return refs
def populate_issue_tree(response_dict, issue_tree):
refs = extract_crossrefs(response_dict)
for num in refs:
def github_json_iter(url, **kwargs):
"""iterate github api url"""
next_url = url
while next_url is not None:
r = requests.get(next_url, **kwargs)
if not r.ok:
raise Exception('Error while making requests')
next_url = r.links['next']['url']
except KeyError:
next_url = None
yield r.json()
if not args.load_intermediate:
for page in github_json_iter("{}/{}/issues".format(args.owner[0],
args.repo[0]), params={ 'access_token' : github_token }):
for issue in page:
# Resolve issue number
issue_num = issue['number']
data['issue_nums'][issue_num] = issue
# Populate the issue tree
populate_issue_tree(issue['body'], data['issue_tree'])
# Parse comments
for comment_page in github_json_iter("{}/{}/issues/{}/comments".format(args.owner[0], args.repo[0], issue['number']),
params={ 'access_token' : github_token }):
for comment in comment_page:
populate_issue_tree(comment['body'], data['issue_tree'])
if args.intermediate_datafile is not None:
print('Saving intermediate file to:', args.intermediate_datafile)
pickle.dump(data, open(args.intermediate_datafile, 'wb'), protocol=pickle.HIGHEST_PROTOCOL)
print('Loading from data from:', args.intermediate_datafile)
data = pickle.load(open(args.intermediate_datafile, 'rb'))
# Dump out a dot file
dot = Digraph(comment="Github Issue Relations: {}/{}".format(args.owner[0], args.repo[0]),
'rankdir' : 'LR',
'splines' : 'curved'
added_nodes = set()
for issue_num, related_set in data['issue_tree'].iteritems():
# Add node
if issue_num not in added_nodes:
dot.node(str(issue_num), data['issue_nums'][issue_num]['title'],
# Add edges
for related_num in related_set:
if related_num not in added_nodes:
dot.node(str(issue_num), data['issue_nums'][related_num]['title'],
dot.edge(str(related_num), str(issue_num))
if args.output[0] != "-":
with open(args.output[0], 'w') as f:
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment