Print language statistics for all ROS packages in the rosdistro

rosdistro_stats.py

#!/usr/bin/env python

# Software License Agreement (BSD License)
#
# Copyright (c) 2018, DFKI GmbH
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above
#    copyright notice, this list of conditions and the following
#    disclaimer in the documentation and/or other materials provided
#    with the distribution.
#  * Neither the name of DFKI GmbH nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

# Author: Martin Guenther <martin.guenther@dfki.de>

"""
Print language statistics for all ROS packages in the rosdistro
"""

from __future__ import division

import json
import sys
from collections import defaultdict
from operator import itemgetter
from urllib2 import urlopen, Request
import yaml

LOAD_FROM_FILE = False
SAVE_TO_FILE = True

if LOAD_FROM_FILE:
    with open('repos.json') as infile:
        repos = json.load(infile)
else:
    # authenticate so we increase the rate limit from 60 to 5000 requests/h
    print "Please generate a personal access token here: https://github.com/settings/tokens/new ."
    print "You don't need to give it any permissions, it is only required to increase the rate limit "
    print "when accessing the GitHub API."
    print
    token = unicode(raw_input('Enter GitHub personal access token: '))

    request = Request(u'https://api.github.com/rate_limit')
    request.add_header('Authorization', 'token %s' % token)
    response = urlopen(request)
    if response.code != 200:
        print 'ERROR: wrong access token'
        sys.exit(-1)

    print '\nWorking, please be patient. This will take 5-10 minutes.'
    print 'Loading rosdistro repos...'
    reponames = []
    for info in yaml.load(urlopen('https://raw.githubusercontent.com/ros/rosdistro/master/kinetic/distribution.yaml'))[
                          'repositories'].itervalues():
        try:
            reponames.append(info['source']['url'])
        except KeyError:
            pass
    print '... done (%d repos loaded).' % len(reponames)

    # reponames = ['https://github.com/ros/kdl_parser.git', ...]

    # filter out non-github repos, remove github prefix + '.git'
    reponames = [r[19:-4] for r in reponames if r.find('https://github.com/') == 0]
    # reponames = ['ros/kdl_parser', ...]

    remaining_rate = json.load(response)['rate']['remaining']
    if len(reponames) > remaining_rate:
        print 'ERROR: more repos to process (%d) than remaining rate (%d)' % (len(reponames), remaining_rate)
        sys.exit(-1)

    repos = []
    i = 0
    for reponame in reponames:
        i += 1
        print 'Reading repo %d/%d...' % (i, len(reponames))
        request = Request(u'https://api.github.com/repos/%s' % reponame)
        request.add_header('Authorization', 'token %s' % token)
        response = urlopen(request)
        if response.code != 200:
            print 'ERROR: rate limited?'
            sys.exit(-1)
        repos.append(json.load(response))

    if SAVE_TO_FILE:
        with open('repos.json', 'w') as outfile:
            json.dump(repos, outfile)

print '\n\n### Languages by popularity\n'

lang_stats = defaultdict(lambda: 0)
for repo in repos:
    lang_stats[repo['language']] += 1

total = sum(lang_stats.itervalues())

print '| rank | Language        | repos | percent |'
print '|------|-----------------|------:|--------:|'
i = 0
for lang in sorted(lang_stats, key=lambda l: lang_stats[l], reverse=True):
    i += 1
    print '| {:3d}. | {:15} | {:5d} | {:6.1f}% |'.format(i, lang, lang_stats[lang], lang_stats[lang] / total * 100)

print '\n\n### ROS repos by popularity\n'

stargazers = [(repo['full_name'], repo['stargazers_count'], repo['language']) for repo in repos]

print '| rank | repo name                                                    | stars | language        |'
print '|------|--------------------------------------------------------------|------:|-----------------|'
i = 0
for item in sorted(stargazers, key=itemgetter(1), reverse=True):
    i += 1
    print '| {:3d}. | {:60} | {:5d} | {:15} |'.format(i, *item)