github_metrics.py

#!/usr/bin/env python
'''Script to pull data out of GitHub and push into Elasticsearch'''

import os
import sys
import requests
import httplib
import json  # NOQA
from urlparse import urljoin
from uritemplate import expand
from requests.packages.urllib3.exceptions import InsecureRequestWarning

# Disable warnings
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

for name in ('GITHUB_AUTH_USER', 'GITHUB_AUTH_TOKEN'):
    if name not in os.environ:
        print("Error!  Please set the environment variable {0}".format(name))
        sys.exit(1)

auth = (os.environ['GITHUB_AUTH_USER'], os.environ['GITHUB_AUTH_TOKEN'])
base_url = 'https://api.github.com'
elastic_url = 'http://{0}:{1}/'.format(
    os.environ.get('ELASTIC_HOST', 'localhost'),
    os.environ.get('ELASTIC_PORT', '9200')
)


def request(url, **kwargs):
    r = requests.get(url, verify=False, auth=auth, **kwargs)
    assert r.status_code == httplib.OK, "Unexpected response code (%s != %s)" % (r.status_code, httplib.OK)
    return r


def get_issues(url, **kwargs):
    r = request(url, params=kwargs)
    issues = r.json()

    # Handle pagination
    while 'next' in r.links:
        r = request(r.links['next']['url'])
        issues.extend(r.json())

    return issues

if __name__ == '__main__':
    r = request(urljoin(base_url, '/repos/ansible/ansible-tower'))
    repository = r.json()
    issues_url = expand(repository['issues_url'])
    milestones_url = expand(repository['milestones_url'])

    # Get all issues
    issues = get_issues(issues_url, state="all")
    issues = filter(lambda x: 'pull_request' not in x, issues)
    print "Collected %s issues" % len(issues)
    for issue in issues:
        r = requests.put(urljoin(elastic_url, "/ansible-tower/issues/%s" % issue['number']), data=json.dumps(issue))
        print(" * [{status_code}] {url}".format(**r.__dict__))
        assert r.status_code in (httplib.CREATED, httplib.OK), "Unexpected status_code [%s] %s" % (r.status_code, r.text)