bazelexternals -- Bazel externals extraction for Ebuild

bazelexternals.py

#!/usr/bin/python3
"""A program to extract externals from a Bazel workspace.

Example:

  bazelexternals mysystem-1.0.ebuild .

To update mysystem-1.0.ebuild with externals from the workspace in CWD.
The ebuild file must have a line like

  bazel_uri="..."

possibly spanning multiple lines. This will be replaced with new sources. A file
called files/bazel-markers-1.0.patch will be created that contains the marker
files in the Bazel base directory, needed because generating the fingerprints
is difficult from outside the Bazel code and markers may contain arbitrary
information.
"""

import argparse
import collections
import difflib
import hashlib
import os.path
import shutil
import subprocess
import sys
import tempfile
import xml.etree.ElementTree as ET

import portage.versions as versions


HttpArchive = collections.namedtuple("HttpArchive", ["name", "urls", "strip_prefix", "marker", "build_file"])
HttpFile = collections.namedtuple("HttpFile", ["name", "urls", "strip_prefix", "marker"])


def marker_name(name):
    return "@{}.marker".format(name.replace("//external:", ""))


def label_to_path(label):
    return label.lstrip("/").replace(":", os.path.sep)


def uri_record(c):
    args = [c.urls[0]]
    mrk = "=>"
    if isinstance(c, HttpFile):
        mrk = "->"
    args.extend([mrk, c.name.replace("//external:", ""), c.strip_prefix or ""])
    if isinstance(c, HttpArchive) and c.build_file:
        args.append(label_to_path(c.build_file))
    return " ".join(args)


def main():
    argp = argparse.ArgumentParser()
    argp.add_argument("ebuild", type=str, help="Ebuild file to update")
    argp.add_argument("workspace", type=str, help="Bazel workspace directory")
    args = argp.parse_args()
    print(args)

    targets = ["//tensorflow/tools/pip_package:build_pip_package", "//tensorflow:libtensorflow.so"]

    subprocess.check_call(["bazel", "fetch"] + targets, cwd=args.workspace)

    output_base = subprocess.check_output(["bazel", "info", "output_base"], cwd=args.workspace).decode("utf-8").rstrip()
    extdeps = set()
    for l in subprocess.check_output([
            "bazel",
            "query", "--output", "label", "--nofetch",
            "filter('//external:|@', deps({}))".format(" union ".join(targets))], cwd=args.workspace).splitlines():
        l = l.decode("utf-8").strip("'").strip()
        if l.startswith("//external:"):
            extdeps.add(l)
        else:
            extdeps.add("//external:" + l[1:].split("//")[0])

    xml = subprocess.check_output([
        "bazel",
        "query", "--output", "xml", "--nofetch",
        "kind('git_repository|http_archive|http_file|maven_jar|maven_server rule', //external:*)"], cwd=args.workspace)
    rules = ET.fromstring(xml)
    deps = []
    for r in rules.iter("rule"):
        name = r.get("name")
        if name not in extdeps:
            continue

        sp = r.find("string[@name='strip_prefix']")
        sp = sp.get("value") if sp is not None else None
        build_file = r.find("string[@name='build_file']")
        build_file = build_file.get("value") if build_file is not None else None
        if not build_file:
            build_file = r.find("label[@name='build_file']")
            build_file = build_file.get("value") if build_file is not None else None
        if r.get("class") in ("http_archive", "http_file", "new_http_archive", "temp_workaround_http_archive"):
            urls = []
            for u in r.findall("list[@name='urls']/string"):
                urls.append(u.get("value"))
            for u in r.findall("string[@name='url']"):
                urls.append(u.get("value"))

            with open(os.path.join(output_base, "external", marker_name(name)), "r") as f:
                marker = f.read().splitlines()

            if r.get("class") == "http_file":
                deps.append(HttpFile(name, urls, sp, marker))
            else:
                deps.append(HttpArchive(name, urls, sp, marker, build_file))
        else:
            raise ValueError("unimplemented rule type {}".format(r.get("class")))

    deps = sorted(deps, key=lambda x: x.name)

    if not os.path.exists(args.ebuild):
        raise ValueError("ebuild file does not exist")

    pn, pv, r = versions.pkgsplit(os.path.basename(args.ebuild)[:-7])

    # Update the ebuild source URIs.
    try:
        with open(args.ebuild, "rt") as inf, tempfile.NamedTemporaryFile('wt', prefix=os.path.basename(args.ebuild), dir=os.path.dirname(args.ebuild), delete=False) as outf:
            discard = False
            for l in inf:
                if l.strip().startswith("bazel_uri=\""):
                    print("bazel_uri=\"" + "\n           ".join(uri_record(d) for d in deps) + "\"", file=outf)
                    discard = True
                if not discard:
                    print(l.rstrip(), file=outf)
                if discard and l.rstrip().endswith("\""):
                    discard = False

        shutil.copymode(args.ebuild, outf.name)
    except:
        os.unlink(outf.name)
        raise
    else:
        os.rename(outf.name, args.ebuild)

    # Create a patch file with markers.
    filesdir = os.path.join(os.path.dirname(args.ebuild), "files")
    if not os.path.exists(filesdir):
        os.makedirs(filesdir)
    markerfile = os.path.join(filesdir, "bazel-markers-{}.patch".format(pv))
    try:
        with open(markerfile, "wt") as outf:
            for d in deps:
                outf.writelines(difflib.unified_diff(b"", d.marker, fromfile=os.path.join("bazel-base", "external", marker_name(d.name)), tofile=os.path.join("bazel-base", "external", marker_name(d.name))))
                print(file=outf)
    except:
        os.unlink(markerfile)
        raise


if __name__ == "__main__":
    main()

Dockerfile

FROM gentoo/stage3-amd64

WORKDIR /
ADD package.accept_keywords etc/portage/

RUN wget http://distfiles.gentoo.org/snapshots/portage-latest.tar.bz2 && \
    bzcat /portage-latest.tar.bz2 | tar -xf - -C /usr && \
    mkdir -p /usr/portage/distfiles /usr/portage/metadata /usr/portage/packages && \
    rm /portage-latest.tar.bz2

RUN emerge cpuid2cpuflags && \
    rm -f /usr/portage/distfiles/*

# Tensorflow warns in runtime if there are unused possible optimizations.
# ... and we want good performance.
RUN sed -i -e '/^CPU_FLAGS_X86=/d' /etc/portage/make.conf && \
    cpuinfo2cpuflags-x86 >>/etc/portage/make.conf && \
    echo 'MAKEOPTS="-j"' >>/etc/portage/make.conf

# zip is a workaround for a Bazel 0.4.4 ebuild bug.
RUN USE='-X -cups -alsa -webstart -gtk -ncurses headless-awt' emerge \
    dev-python/six \
    dev-python/wheel \
    zip \
    bazel \
    unzip \
    '>=dev-python/numpy-1.11.0' \
    '>=dev-python/protobuf-python-3.1.0' \
    dev-lang/go && \
    rm -f /usr/portage/distfiles/*

ADD sci-lib/ /usr/local/portage/sci-lib/

# Need /etc/profile to set JAVA_HOME.
RUN source /etc/profile && \
    USE='go lib python' ebuild --skip-manifest /usr/local/portage/sci-lib/tensorflow/tensorflow-1.0.1.ebuild merge && \
    rm -f /usr/portage/distfiles/* /var/tmp/portage/*