Github Issue Relation downloader

issue-graph.py

#!/usr/bin/env python

from __future__ import print_function

import os
import sys
import requests
import keyring
import getpass

import argparse
import re
import pickle

from collections import defaultdict

from graphviz import Digraph

UUID="com.wrouesnel.github.issue-graph"
USER="github_token"

parser = argparse.ArgumentParser(description="Parse issue comments and build a reference tree",
                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--new-token', action='store_true', help='Change Github API token')
parser.add_argument('owner', nargs=1, help='owner for github repo to inspect')
parser.add_argument('repo', nargs=1, help='repository under github owner to inspect')

parser.add_argument('--output', '-o', dest='output', nargs=1, default='-', help='Output destination')

parser.add_argument('--intermediate', dest='intermediate_datafile', default=None, help='If specified, save API data locally.')
parser.add_argument('--local-load', dest='load_intermediate', action="store_true", help='Do not contact API, use --intermediate pickle file.')

args = parser.parse_args()

github_token = keyring.get_password(UUID, USER)
if (github_token is None) or args.new_token:
    new_token = getpass.getpass('Enter Github API Key (characters will not be shown):')
    keyring.set_password(UUID, USER, new_token)
    print("New token set.")
    github_token = new_token
    
if github_token is None:
    raise Exception("No Github API token set.")

next_url = "https://api.github.com/repos/{}/{}/issues".format(args.owner[0], args.repo[0])

# Tree of discovered issues stored in target <- sources target form, deduped.
data = {
    'issue_tree' : defaultdict(set),
    'issue_nums' : dict()
}

rx = re.compile('''#\d+''')
def extract_crossrefs(body):
    refs = []
    m = rx.findall(body)
    for s in m:
        refs.append(int(s.replace("#","")))
    return refs

def populate_issue_tree(response_dict, issue_tree):
    refs = extract_crossrefs(response_dict)
    for num in refs:
        issue_tree[num].add(int(issue_num))

def github_json_iter(url, **kwargs):
    """iterate github api url"""
    next_url = url
    while next_url is not None:
        r = requests.get(next_url, **kwargs)
        if not r.ok:
            raise Exception('Error while making requests')
        try:
            next_url = r.links['next']['url']
        except KeyError:
            next_url = None
        yield r.json()

if not args.load_intermediate:
    for page in github_json_iter("https://api.github.com/repos/{}/{}/issues".format(args.owner[0], 
    args.repo[0]), params={ 'access_token' : github_token }):
        for issue in page:
            # Resolve issue number
            issue_num = issue['number']
            data['issue_nums'][issue_num] = issue
            # Populate the issue tree
            populate_issue_tree(issue['body'], data['issue_tree'])
            # Parse comments
            for comment_page in github_json_iter("https://api.github.com/repos/{}/{}/issues/{}/comments".format(args.owner[0], args.repo[0], issue['number']),
            params={ 'access_token' : github_token }):
                for comment in comment_page:
                    populate_issue_tree(comment['body'], data['issue_tree'])
                    
    if args.intermediate_datafile is not None:
        print('Saving intermediate file to:', args.intermediate_datafile)
        pickle.dump(data, open(args.intermediate_datafile, 'wb'), protocol=pickle.HIGHEST_PROTOCOL)
else:
    print('Loading from data from:', args.intermediate_datafile)
    data = pickle.load(open(args.intermediate_datafile, 'rb'))

# Dump out a dot file
dot = Digraph(comment="Github Issue Relations: {}/{}".format(args.owner[0], args.repo[0]),
    graph_attr={
        'rankdir' : 'LR',
        'splines' : 'curved'
    })

added_nodes = set()
for issue_num, related_set in data['issue_tree'].iteritems():
    # Add node
    if issue_num not in added_nodes:
        dot.node(str(issue_num), data['issue_nums'][issue_num]['title'],
            shape='box')
    # Add edges
    for related_num in related_set:
        if related_num not in added_nodes:
            dot.node(str(issue_num), data['issue_nums'][related_num]['title'],
                shape='box')            
        dot.edge(str(related_num), str(issue_num))

if args.output[0] != "-":
    with open(args.output[0], 'w') as f:
        f.write(dot.source)
        f.write("\n")
else:
    sys.stdout.write(dot.source)
    sys.stdout.write("\n")