Python GoDeps extractor

licenses.py

import json
from pprint import pprint

import urllib.request

REWRITE = {
    'cloud.google.com/go': 'github.com/GoogleCloudPlatform/gcloud-golang',
    'golang.org/x':  'github.com/golang',
    'k8s.io': 'github.com/kubernetes',
    'gopkg.in/yaml.v2': 'github.com/go-yaml/yaml',
    'gopkg.in/inf.v0': 'github.com/go-inf/inf'
}


def rewrite(url):
    for k, v in REWRITE.items():
        if url.startswith(k):
            url = url.replace(k, v)
    return url


def get_root_package(path):
    path = path.replace('github.com', '')
    root_package = path.split('/')[1:3]
    return '/'.join(root_package)


def get_github_raw_url(path, rev):
    root = get_root_package(path)
    return 'https://raw.githubusercontent.com/%s/%s/LICENSE' % (root, rev)


def get_licenses():
    results = []
    with open('GoDeps/GoDeps.json') as f:
        data = json.load(f)
        deps = data['Deps']
        for dep in deps:
            path = rewrite(dep['ImportPath'])
            rev = dep['Rev']
            results.append(get_github_raw_url(path, rev))
    return set(results)


def process_deps():
    with open('GoDeps/GoDeps.json') as f:
        data = json.load(f)
        deps = data['Deps']
        for dep in deps:
            path = rewrite(dep['ImportPath'])
            rev = dep['Rev']
            if 'Comment' in dep:
                comment = dep['Comment']
                print("%s %s %s" % (path, rev, comment))
            else:
                print("%s %s" % (path, rev))


def main():
    results = get_licenses()
    with open('results.txt', 'wb') as f:
        for result in results:
            try:
                content = urllib.request.urlopen(result).read()
                f.write(content)
            except Exception as e:
                try:
                    content = urllib.request.urlopen(result + '.txt').read()
                    f.write(content)
                except Exception as e:
                    print(result)


if __name__ == '__main__':
    main()

script.py

import json
from pprint import pprint

def main():
    with open('GoDeps/GoDeps.json') as f:
        data = json.load(f)
        deps = data['Deps']
        for dep in deps:
            path = dep['ImportPath']
            rev = dep['Rev']
            if 'Comment' in dep:
                comment = dep['Comment']
                print("%s %s %s" % (path, rev, comment))
            else:
                print("%s %s" % (path, rev))

if __name__ == '__main__':
    main()