download-jar.py

#!/usr/bin/env python

import sys
import os
import argparse
from os import path
from os.path import isfile as exists
import signal
import json
import xml.etree.ElementTree as ET

def memoize(f):
    class memodict(dict):
        def __missing__(self, key):
            ret = self[key] = f(key)
            return ret
    return memodict().__getitem__

def signal_handler(signal, frame):
    sys.exit(0)

signal.signal(signal.SIGINT, signal_handler)

def mkdir(d):
    print "mkdir:", d
    os.system("mkdir -p " + d)

def download(dst, url):
    print "downloading:", url
    os.system("curl -s -L -o %s %s" % (dst, url))

def generate_deps(artifact, pom):
    print "generating dependencies from:", pom
    output_file = '.%s.deps.tgf' % (artifact,)
    tmp_file = artifact + '/' + output_file
    if os.system("which mvn >> /dev/null") != 0:
        print "Please install mvn"
        sys.exit(1)
    # Small hack, as we don't know how to handle bundle packaging
    os.system("sed -i 's#<packaging>bundle</packaging>#<packaging>jar</packaging>#' %s" % (pom,))
    # Small hack, as we don't know how to handle zip packaging
    os.system("sed -i 's#<packaging>zip</packaging>#<packaging>jar</packaging>#' %s" % (pom,))
    os.system("mvn -f %s dependency:tree -DoutputFile=%s -DoutputType=tgf" % (pom, output_file))
    # Read POM file so we can exclude optionals
    flatten = lambda l: [item for sublist in l for item in sublist]
    ns = {'pom': 'http://maven.apache.org/POM/4.0.0'}
    pom_deps = flatten([d.getchildren() for d in ET.parse(pom).getroot().findall('pom:dependencies', ns)])
    jar_mapping = {}
    compile_deps = []
    test_deps = []
    with open(tmp_file) as f:
        this_jar, _ = f.readline().strip().split(' ')
        for line in f:
            if line.strip() == '#':
                break
            jar_id, artifact = line.strip().split(' ')
            # Fixing edge case with multiple packages specified
            if artifact.count(':') != 4 and artifact.endswith('test'):
                # skip test deps
                continue
            elif artifact.count(':') != 4:
                print "bad artifact format:", artifact
            group, name, _, version, _ = artifact.split(':')
            # By default, don't download optional deps
            # You must explictly list them to download
            optional = False
            for d in pom_deps:
                if d.find('pom:groupId', ns).text == group and \
                   d.find('pom:artifactId', ns).text == name:
                    optional_element = d.find('pom:optional', ns)
                    optional = optional_element is not None and optional_element.text == 'true'
                    break
            if not optional:
                jar_mapping[jar_id] = ':'.join([group, name, version])
            else:
                print "skipping optional dep:", artifact
        for line in f:
            parent, child, scope = line.strip().split(' ')
            if parent != this_jar:
                continue
            if child not in jar_mapping:
                continue
            if scope == "test":
                test_deps.append(jar_mapping[child])
            elif scope == "compile":
                compile_deps.append(jar_mapping[child])
            else:
                print "unknown scope for dependancy:",jar_mapping[child], scope
    os.system("rm " + tmp_file)
    return compile_deps

def generate_build(artifact, compile_deps, build_file):
    deps = ["//thirdparty/" + d for d in compile_deps]
    build_rule = """
java_import(
  name = "%s",
  jars = ["%s.jar"],
  visibility = ['//visibility:public'],
  deps = %s)""" % (
      artifact,
      artifact,
      json.dumps(deps, indent=4, sort_keys=True, separators=(',', ':'))
  )
    with open(build_file, 'w') as f:
        f.truncate()
        f.write(build_rule)

@memoize
def download_jar(jar):
    BASE_URL = "http://central.maven.org/maven2"
    parts = jar.split(':')
    parts[0] = parts[0].replace('.', '/')
    artifact = parts[1]
    version = parts[2]

    mkdir(artifact)

    file_tuple = (artifact, artifact)
    url_tuple = (BASE_URL, '/'.join(parts), artifact, version)

    pom_file = "%s/%s.pom" % file_tuple
    jar_file = "%s/%s.jar" % file_tuple
    build_file = "%s/BUILD" % (artifact,)

    if exists(pom_file) and exists(jar_file):
        print "skipping downloaded artifact:", artifact
    else:
        download(pom_file, "%s/%s/%s-%s.pom" % url_tuple)
        download(jar_file, "%s/%s/%s-%s.jar" % url_tuple)
        download("%s/%s-javadoc.jar" % file_tuple, "%s/%s/%s-%s-javadoc.jar" % url_tuple)
        download("%s/%s-sources.jar" % file_tuple, "%s/%s/%s-%s-sources.jar" % url_tuple)

    if not exists(build_file):
        deps = generate_deps(artifact, pom_file)
        deps = [download_jar(dep) for dep in deps]
        generate_build(artifact, deps, build_file)

    return artifact

if __name__ == "__main__":
    if not exists("WORKSPACE"):
        print "Please run from project root directory"
        sys.exit(1)
    parser = argparse.ArgumentParser(description="Download jars into //thirdparty")
    parser.add_argument("-j", dest="jar", help="Ex: com.google.guava:guava:20.0")
    parser.add_argument("-d", dest="dependencies", help="install from a dependencies file")
    args = parser.parse_args()
    mkdir("thirdparty")
    if args.jar and args.dependencies:
        print "Make up your mind, you can only use one parameter at a time"
        sys.exit(1)
    elif not args.jar and not args.dependencies:
        print "You must specify at least one parameter!"
        sys.exit(1)
    elif args.dependencies:
        with open(args.dependencies) as f:
            os.chdir("thirdparty")
            for dependency in f:
                download_jar(dependency.strip())
    else:
        os.chdir("thirdparty")
        artifact = download_jar(args.jar)
        print "You can now use the target:", "//thirdparty/" + artifact