quick & dirty node.js dependency analysis tools

countbugs.py

#!/usr/bin/env python3

# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json, os, sys, re, requests

node_modules = os.path.join(sys.argv[1], "node_modules/")

packages = []
for root, dirs, files in os.walk(node_modules):
    for f in files:
        if f == "package.json":
            packages.append(os.path.join(root, f))

packages_with_names = []
for package in packages:
    packages_with_names.append((package, "/".join(package.split("/")[-2])))

seen_packages = []
uniq_packages = []
for package in packages_with_names:
    if package[1] not in seen_packages:
        seen_packages.append(package[1])
        uniq_packages.append(package[0])

repo_regex = re.compile("github\.com[/:]([a-zA-Z0-9-_]+/[a-zA-Z0-9-_\.]+)")

repos = []
for package in uniq_packages:
    try:
        with open(package) as p:
            repo_url = json.load(p)["repository"]["url"]
            repo = repo_regex.search(repo_url).group(1)
            if repo[-4:] == ".git":
                repo = repo[:-4]
            repos.append(repo)
    except KeyError:
        pass
    except AttributeError:
        pass

def _get_num_bugs_internal(link, oauth_token):
    next_capture = re.compile("<([^>]+)>; rel=\"next\"")
    r = requests.get(link, {'access_token': oauth_token, 'per_page': 100})
    if r.status_code == 200:
        n = 0
        for item in r.json():
            if not "pull_request" in item:
                n += 1
        if "Link" in r.headers and 'rel="next"' in r.headers["Link"]:
            next_link = next_capture.search(r.headers["Link"]).group(1)
            return n + _get_num_bugs_internal(next_link, oauth_token)
        else:
            return n
    else:
        print("\nError getting github issues for {}: {}\n{}".format(repo, r.status_code, r.text))
        return None

def get_num_bugs(repo, oauth_token):
    return _get_num_bugs_internal("https://api.github.com/repos/{}/issues".format(repo), oauth_token)

total_bugs = 0
for i, repo in enumerate(repos):
    print("\rFetching bugs for repo #{}/{}".format(i, len(repos)), end="")
    sys.stdout.flush()
    bugs = get_num_bugs(repo, os.environ["GH_OAUTH_TOKEN"])
    if bugs is not None:
        total_bugs += bugs

print("\nThere are {} bugs in the dependency tree of {}".format(total_bugs, sys.argv[1]))

countmaintainers.js

// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Warning! This currently hits the npm API way too hard - it should be rate
// limited, but it worked for what I needed, so I haven't gotten around to it.

var api = require('api-npm');

var all_maintainers = []

function append_maintainers(data)
{
	if (data.hasOwnProperty("maintainers")) {
		data.maintainers.forEach(function (val, index, array) {
			all_maintainers.push(val.name);
		});
	}
	n -= 1;
	if (n == 0) {
		all_maintainers = all_maintainers.filter(function(item, pos){
			return all_maintainers.indexOf(item) == pos; 
		});
		console.log(all_maintainers.length);
	}
}

var args = process.argv.slice(2);

var n = args.length;

args.forEach(function (val, index, array) {
	api.getdetails(val, append_maintainers);
});