Scrapes github issues for complexity points and business value points from the issue markup (and also sets the title of the issue to include these points). In the end it prints out the issues that are completely filled out in a CSV format.

scraper.py

import json
import re
import requests

# Set USERNAME and PASSWORD to your Github credentials and REPO_URL to your github repo.
import settings

auth = (settings.USERNAME, settings.PASSWORD)
api_url = 'https://api.github.com/repos/%s/%%s' % (settings.REPO_URL, )

class AuthenticationException(Exception): pass

def dump_issue(issue):
	if 'business_value' not in issue:
		issue['business_value'] = 21
	if 'complexity' not in issue:
		issue['complexity'] = '-'

	row = u'"#%s","%d","%s","%s"' % (
			issue['number'],
			issue['business_value'],
			issue['complexity'],
			issue['title'].replace('"', "'"),
			)

	print row.encode('utf8')

with requests.session(auth=auth) as c:
	def get(path, *args, **kwargs):
		data = dict(per_page=200)
		data.update(kwargs.pop('data', {}))
		r = c.get(api_url % path, *args, data=data, **kwargs)
		response = json.loads(r.content)
		if 'message' in response and response['message'] == u"Bad credentials":
			raise AuthenticationException("Bad Github credentials")
		return response

	def patch(url, data):
		r = c.post(url, data=json.dumps(data))
		return r

	q = dict(labels='epic')
	epics = get('issues', data=q)
	re_complexity = re.compile(r':sparkles:(\d+)')
	re_total_complexity = re.compile(r'Totalt?:\s*:sparkles:(\d+)')
	re_business_value = re.compile(r'(Business value|BV): (\d+)')

	complete = []
	not_complete = []

	for epic in epics:
		title = epic['title']
		body = " ".join(epic['body'].splitlines())
		for match in re_total_complexity.finditer(body):
			epic['complexity'] = int(match.group(1))

		# Remove total complexity matches so that it isn't counted again
		body = re_total_complexity.sub('', body)

		for match in re_complexity.finditer(body):
			if 'complexities' not in epic:
				epic['complexities'] = []
			epic['complexities'].append(int(match.group(1)))
		for match in re_business_value.finditer(body):
			if 'business_value' in epic:
				print "Warning: Business value defined multiple times in #%s" % epic['number']
			epic['business_value'] = int(match.group(2))

		if 'complexity' not in epic and 'complexities' in epic:
			epic['complexity'] = sum(epic['complexities'])

		if 'complexity' in epic and 'business_value' in epic:
			if 'complexities' in epic:
				complexity = sum(epic['complexities'])
				if complexity != epic['complexity']:
					print "#%s has mismatched complexity count: %d instead of %d" % (epic['number'], epic['complexity'], complexity)

			complete.append(epic)

			title = epic['title']
			# Remove old complexity from title
			title = re.sub(r'\[\d+\]', '', title)
			# Remove old BV from title
			title = re.sub(r'{\d+}', '', title)

			# Reformat title
			title = "%s [%d] {%d}" % (title.strip(), epic['complexity'], epic['business_value'], )

			# Update issue if the title has changed
			if title != epic['title']:
				print "Updating title for: #%s" % (epic['number'])
				patch(epic['url'], data={
						'title': title,
					})
				epic['title'] = title
		else:
			not_complete.append(epic)

	print "THESE ARE NOT COMPLETE!"
	for epic in not_complete:
		print "#%s: %s" % (epic['number'], epic['url'].replace('api.github.com/repos', 'github.com'))

	print "--"
	# print "THESE ARE COMPLETE!"
	# for epic in complete:
	#     print u"#%s: %r" % (epic['number'], epic['title'])

	for epic in complete:
		dump_issue(epic)

	high_priority_issues = get('issues', data={'labels': 'regression,bug,quick-fix'})
	for issue in high_priority_issues:
		dump_issue(issue)