Skip to content

Instantly share code, notes, and snippets.

@rockwotj

rockwotj/download-jar.py

Last active Jan 7, 2017
Embed
What would you like to do?
#!/usr/bin/env python
import sys
import os
import argparse
from os import path
from os.path import isfile as exists
import signal
import json
import xml.etree.ElementTree as ET
def memoize(f):
class memodict(dict):
def __missing__(self, key):
ret = self[key] = f(key)
return ret
return memodict().__getitem__
def signal_handler(signal, frame):
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
def mkdir(d):
print "mkdir:", d
os.system("mkdir -p " + d)
def download(dst, url):
print "downloading:", url
os.system("curl -s -L -o %s %s" % (dst, url))
def generate_deps(artifact, pom):
print "generating dependencies from:", pom
output_file = '.%s.deps.tgf' % (artifact,)
tmp_file = artifact + '/' + output_file
if os.system("which mvn >> /dev/null") != 0:
print "Please install mvn"
sys.exit(1)
# Small hack, as we don't know how to handle bundle packaging
os.system("sed -i 's#<packaging>bundle</packaging>#<packaging>jar</packaging>#' %s" % (pom,))
# Small hack, as we don't know how to handle zip packaging
os.system("sed -i 's#<packaging>zip</packaging>#<packaging>jar</packaging>#' %s" % (pom,))
os.system("mvn -f %s dependency:tree -DoutputFile=%s -DoutputType=tgf" % (pom, output_file))
# Read POM file so we can exclude optionals
flatten = lambda l: [item for sublist in l for item in sublist]
ns = {'pom': 'http://maven.apache.org/POM/4.0.0'}
pom_deps = flatten([d.getchildren() for d in ET.parse(pom).getroot().findall('pom:dependencies', ns)])
jar_mapping = {}
compile_deps = []
test_deps = []
with open(tmp_file) as f:
this_jar, _ = f.readline().strip().split(' ')
for line in f:
if line.strip() == '#':
break
jar_id, artifact = line.strip().split(' ')
# Fixing edge case with multiple packages specified
if artifact.count(':') != 4 and artifact.endswith('test'):
# skip test deps
continue
elif artifact.count(':') != 4:
print "bad artifact format:", artifact
group, name, _, version, _ = artifact.split(':')
# By default, don't download optional deps
# You must explictly list them to download
optional = False
for d in pom_deps:
if d.find('pom:groupId', ns).text == group and \
d.find('pom:artifactId', ns).text == name:
optional_element = d.find('pom:optional', ns)
optional = optional_element is not None and optional_element.text == 'true'
break
if not optional:
jar_mapping[jar_id] = ':'.join([group, name, version])
else:
print "skipping optional dep:", artifact
for line in f:
parent, child, scope = line.strip().split(' ')
if parent != this_jar:
continue
if child not in jar_mapping:
continue
if scope == "test":
test_deps.append(jar_mapping[child])
elif scope == "compile":
compile_deps.append(jar_mapping[child])
else:
print "unknown scope for dependancy:",jar_mapping[child], scope
os.system("rm " + tmp_file)
return compile_deps
def generate_build(artifact, compile_deps, build_file):
deps = ["//thirdparty/" + d for d in compile_deps]
build_rule = """
java_import(
name = "%s",
jars = ["%s.jar"],
visibility = ['//visibility:public'],
deps = %s)""" % (
artifact,
artifact,
json.dumps(deps, indent=4, sort_keys=True, separators=(',', ':'))
)
with open(build_file, 'w') as f:
f.truncate()
f.write(build_rule)
@memoize
def download_jar(jar):
BASE_URL = "http://central.maven.org/maven2"
parts = jar.split(':')
parts[0] = parts[0].replace('.', '/')
artifact = parts[1]
version = parts[2]
mkdir(artifact)
file_tuple = (artifact, artifact)
url_tuple = (BASE_URL, '/'.join(parts), artifact, version)
pom_file = "%s/%s.pom" % file_tuple
jar_file = "%s/%s.jar" % file_tuple
build_file = "%s/BUILD" % (artifact,)
if exists(pom_file) and exists(jar_file):
print "skipping downloaded artifact:", artifact
else:
download(pom_file, "%s/%s/%s-%s.pom" % url_tuple)
download(jar_file, "%s/%s/%s-%s.jar" % url_tuple)
download("%s/%s-javadoc.jar" % file_tuple, "%s/%s/%s-%s-javadoc.jar" % url_tuple)
download("%s/%s-sources.jar" % file_tuple, "%s/%s/%s-%s-sources.jar" % url_tuple)
if not exists(build_file):
deps = generate_deps(artifact, pom_file)
deps = [download_jar(dep) for dep in deps]
generate_build(artifact, deps, build_file)
return artifact
if __name__ == "__main__":
if not exists("WORKSPACE"):
print "Please run from project root directory"
sys.exit(1)
parser = argparse.ArgumentParser(description="Download jars into //thirdparty")
parser.add_argument("-j", dest="jar", help="Ex: com.google.guava:guava:20.0")
parser.add_argument("-d", dest="dependencies", help="install from a dependencies file")
args = parser.parse_args()
mkdir("thirdparty")
if args.jar and args.dependencies:
print "Make up your mind, you can only use one parameter at a time"
sys.exit(1)
elif not args.jar and not args.dependencies:
print "You must specify at least one parameter!"
sys.exit(1)
elif args.dependencies:
with open(args.dependencies) as f:
os.chdir("thirdparty")
for dependency in f:
download_jar(dependency.strip())
else:
os.chdir("thirdparty")
artifact = download_jar(args.jar)
print "You can now use the target:", "//thirdparty/" + artifact
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment