Last active
April 20, 2017 11:41
-
-
Save tommie/aa0c5495c143bfff347b384b01835d91 to your computer and use it in GitHub Desktop.
bazelexternals -- Bazel externals extraction for Ebuild
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python3 | |
"""A program to extract externals from a Bazel workspace. | |
Example: | |
bazelexternals mysystem-1.0.ebuild . | |
To update mysystem-1.0.ebuild with externals from the workspace in CWD. | |
The ebuild file must have a line like | |
bazel_uri="..." | |
possibly spanning multiple lines. This will be replaced with new sources. A file | |
called files/bazel-markers-1.0.patch will be created that contains the marker | |
files in the Bazel base directory, needed because generating the fingerprints | |
is difficult from outside the Bazel code and markers may contain arbitrary | |
information. | |
""" | |
import argparse | |
import collections | |
import difflib | |
import hashlib | |
import os.path | |
import shutil | |
import subprocess | |
import sys | |
import tempfile | |
import xml.etree.ElementTree as ET | |
import portage.versions as versions | |
HttpArchive = collections.namedtuple("HttpArchive", ["name", "urls", "strip_prefix", "marker", "build_file"]) | |
HttpFile = collections.namedtuple("HttpFile", ["name", "urls", "strip_prefix", "marker"]) | |
def marker_name(name): | |
return "@{}.marker".format(name.replace("//external:", "")) | |
def label_to_path(label): | |
return label.lstrip("/").replace(":", os.path.sep) | |
def uri_record(c): | |
args = [c.urls[0]] | |
mrk = "=>" | |
if isinstance(c, HttpFile): | |
mrk = "->" | |
args.extend([mrk, c.name.replace("//external:", ""), c.strip_prefix or ""]) | |
if isinstance(c, HttpArchive) and c.build_file: | |
args.append(label_to_path(c.build_file)) | |
return " ".join(args) | |
def main(): | |
argp = argparse.ArgumentParser() | |
argp.add_argument("ebuild", type=str, help="Ebuild file to update") | |
argp.add_argument("workspace", type=str, help="Bazel workspace directory") | |
args = argp.parse_args() | |
print(args) | |
targets = ["//tensorflow/tools/pip_package:build_pip_package", "//tensorflow:libtensorflow.so"] | |
subprocess.check_call(["bazel", "fetch"] + targets, cwd=args.workspace) | |
output_base = subprocess.check_output(["bazel", "info", "output_base"], cwd=args.workspace).decode("utf-8").rstrip() | |
extdeps = set() | |
for l in subprocess.check_output([ | |
"bazel", | |
"query", "--output", "label", "--nofetch", | |
"filter('//external:|@', deps({}))".format(" union ".join(targets))], cwd=args.workspace).splitlines(): | |
l = l.decode("utf-8").strip("'").strip() | |
if l.startswith("//external:"): | |
extdeps.add(l) | |
else: | |
extdeps.add("//external:" + l[1:].split("//")[0]) | |
xml = subprocess.check_output([ | |
"bazel", | |
"query", "--output", "xml", "--nofetch", | |
"kind('git_repository|http_archive|http_file|maven_jar|maven_server rule', //external:*)"], cwd=args.workspace) | |
rules = ET.fromstring(xml) | |
deps = [] | |
for r in rules.iter("rule"): | |
name = r.get("name") | |
if name not in extdeps: | |
continue | |
sp = r.find("string[@name='strip_prefix']") | |
sp = sp.get("value") if sp is not None else None | |
build_file = r.find("string[@name='build_file']") | |
build_file = build_file.get("value") if build_file is not None else None | |
if not build_file: | |
build_file = r.find("label[@name='build_file']") | |
build_file = build_file.get("value") if build_file is not None else None | |
if r.get("class") in ("http_archive", "http_file", "new_http_archive", "temp_workaround_http_archive"): | |
urls = [] | |
for u in r.findall("list[@name='urls']/string"): | |
urls.append(u.get("value")) | |
for u in r.findall("string[@name='url']"): | |
urls.append(u.get("value")) | |
with open(os.path.join(output_base, "external", marker_name(name)), "r") as f: | |
marker = f.read().splitlines() | |
if r.get("class") == "http_file": | |
deps.append(HttpFile(name, urls, sp, marker)) | |
else: | |
deps.append(HttpArchive(name, urls, sp, marker, build_file)) | |
else: | |
raise ValueError("unimplemented rule type {}".format(r.get("class"))) | |
deps = sorted(deps, key=lambda x: x.name) | |
if not os.path.exists(args.ebuild): | |
raise ValueError("ebuild file does not exist") | |
pn, pv, r = versions.pkgsplit(os.path.basename(args.ebuild)[:-7]) | |
# Update the ebuild source URIs. | |
try: | |
with open(args.ebuild, "rt") as inf, tempfile.NamedTemporaryFile('wt', prefix=os.path.basename(args.ebuild), dir=os.path.dirname(args.ebuild), delete=False) as outf: | |
discard = False | |
for l in inf: | |
if l.strip().startswith("bazel_uri=\""): | |
print("bazel_uri=\"" + "\n ".join(uri_record(d) for d in deps) + "\"", file=outf) | |
discard = True | |
if not discard: | |
print(l.rstrip(), file=outf) | |
if discard and l.rstrip().endswith("\""): | |
discard = False | |
shutil.copymode(args.ebuild, outf.name) | |
except: | |
os.unlink(outf.name) | |
raise | |
else: | |
os.rename(outf.name, args.ebuild) | |
# Create a patch file with markers. | |
filesdir = os.path.join(os.path.dirname(args.ebuild), "files") | |
if not os.path.exists(filesdir): | |
os.makedirs(filesdir) | |
markerfile = os.path.join(filesdir, "bazel-markers-{}.patch".format(pv)) | |
try: | |
with open(markerfile, "wt") as outf: | |
for d in deps: | |
outf.writelines(difflib.unified_diff(b"", d.marker, fromfile=os.path.join("bazel-base", "external", marker_name(d.name)), tofile=os.path.join("bazel-base", "external", marker_name(d.name)))) | |
print(file=outf) | |
except: | |
os.unlink(markerfile) | |
raise | |
if __name__ == "__main__": | |
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
FROM gentoo/stage3-amd64 | |
WORKDIR / | |
ADD package.accept_keywords etc/portage/ | |
RUN wget http://distfiles.gentoo.org/snapshots/portage-latest.tar.bz2 && \ | |
bzcat /portage-latest.tar.bz2 | tar -xf - -C /usr && \ | |
mkdir -p /usr/portage/distfiles /usr/portage/metadata /usr/portage/packages && \ | |
rm /portage-latest.tar.bz2 | |
RUN emerge cpuid2cpuflags && \ | |
rm -f /usr/portage/distfiles/* | |
# Tensorflow warns in runtime if there are unused possible optimizations. | |
# ... and we want good performance. | |
RUN sed -i -e '/^CPU_FLAGS_X86=/d' /etc/portage/make.conf && \ | |
cpuinfo2cpuflags-x86 >>/etc/portage/make.conf && \ | |
echo 'MAKEOPTS="-j"' >>/etc/portage/make.conf | |
# zip is a workaround for a Bazel 0.4.4 ebuild bug. | |
RUN USE='-X -cups -alsa -webstart -gtk -ncurses headless-awt' emerge \ | |
dev-python/six \ | |
dev-python/wheel \ | |
zip \ | |
bazel \ | |
unzip \ | |
'>=dev-python/numpy-1.11.0' \ | |
'>=dev-python/protobuf-python-3.1.0' \ | |
dev-lang/go && \ | |
rm -f /usr/portage/distfiles/* | |
ADD sci-lib/ /usr/local/portage/sci-lib/ | |
# Need /etc/profile to set JAVA_HOME. | |
RUN source /etc/profile && \ | |
USE='go lib python' ebuild --skip-manifest /usr/local/portage/sci-lib/tensorflow/tensorflow-1.0.1.ebuild merge && \ | |
rm -f /usr/portage/distfiles/* /var/tmp/portage/* |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment