Build latest GraalVM CE (Java 8 / 11)

buildGraal.py

#!/usr/bin/python3
import os, sys
import platform
import subprocess
import shutil
from argparse import ArgumentParser, REMAINDER

my_env = os.environ.copy()
current_os = platform.system().lower()
version = "python-1.0.5"
script_url="https://gist.githubusercontent.com/lazar-mitrovic/94430814332adae7d5eaf51907b77379/raw/buildGraal.py"

def info(msg):
    msg = "[INFO] " + msg
    if current_os != "windows":
        CYAN='\033[1;36m'
        NC='\033[0m'
        msg = CYAN + msg + NC
    print(msg)

def run(args):
    if isinstance(args, str):
        args = [args]
    args = [j for i in [item.split() for item in args] for j in i]
    retcode = subprocess.call(args, cwd=os.getcwd(), env=my_env)
    if retcode != 0:
        raise Exception("Command invocation failed with return code: {}".format(retcode))

def rm(path):
    try:
        os.remove(path)
    except:
        shutil.rmtree(path)

def install_deps():
    def is_tool(name):
        """Check whether `name` is on PATH."""
        from distutils.spawn import find_executable
        return find_executable(name) is not None

    if current_os == "linux":
            if is_tool("apt"):
                #GraalVM deps
                run("sudo apt -y install build-essential cmake curl python git libxml2 ruby libssl-dev zlib1g-dev xorg-dev libcurl4-openssl-dev libgfortran-9-dev libgomp1 gfortran-9 libxml2 libc++-dev libreadline-dev libpcre3 libpcre3-dev libbz2-dev xz-utils python3-pip")
                #Gluon deps
                run("sudo apt -y install libgtk-3-dev libgdk3.0-cil-dev libpangocairo-1.0-0 libpango1.0-dev libgl1-mesa-dev libatk1.0-dev libcairo-gobject2 libavcodec-dev libavformat-dev libavutil-dev libasound2-dev libpangoft2-1.0-0 libgdk-pixbuf2.0-dev")
            elif is_tool("dnf"):
                info("FEDORA SUPPORT IS WORK IN PROGRESS!")
                info("Dependencies might be missing from this script.")
                #GraalVM deps
                run("sudo dnf install glibc-devel zlib-devel gcc libffi-devel readline-devel")
                run("sudo dnf install python gcc-c++ make")
                #Gluon deps
                run("sudo dnf install alsa-lib-devel freetype-devel glib2-devel gtk3-devel libX11-devel")
                run("sudo dnf install libXtst-devel mesa-libGL-devel pango-devel zlib-devel")
                run("sudo dnf install ffmpeg-devel")
            else:
                info("dnf is not installed, or your distribution is unsupported.")
                info("Run following commands and restart this script:")
                print()
                print("sudo yum install dnf")
                print("sudo dnf install https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm")
    elif current_os == "darwin": # macOS
        info("MacOS SUPPORT IS WORK IN PROGRESS!")
        info("Dependencies might be missing from this script.")
        if not is_tool("brew"):
            info("brew is not installed.")
            info("Run following and restart this script:")
            print("ruby -e \"$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)\"")
            exit(1)
        run("xcode-select --install")
        run("brew install llvm")
    else:
        info("Windows dependencies are not yet supported!")
        exit(1)
    exit(0)

def check_update():
    info("Checking for script update")
    print("Current version: " + version)
    import requests, tempfile, time
    new_script = requests.get("{}?cachebust={}".format(script_url, time.time())).text
    fd, path = tempfile.mkstemp()
    try:
        with os.fdopen(fd, 'w') as tmp:
            tmp.write(new_script)
        from runpy import run_path
        new_version = run_path(path)["version"]
        if new_version != version:
            with open(sys.argv[0], "w") as f:
                f.write(new_script)
            info("Script updated to version: " + new_version)
            info("Please rerun:")
            print(" ".join(sys.argv))
            exit(1)
        else:
            print("Script is up-to-date.")
    finally:
        os.remove(path)

if __name__ == "__main__":
    my_env = os.environ.copy()

    settings = {}
    settings["dev_mode"] = '1'
    settings["java_version"] = '17'
    settings["dynamic_imports"] = '/substratevm,/tools,/sulong,/graal-js,truffleruby,graalpython' # ,graal-nodejs,fastr
    settings["skip_libraries"] = 'polyglot'
    settings["exclude_components"] = 'nju,nic,LibGraal'
    settings["working_dir"] = os.path.abspath(os.path.join(os.environ['GSC'], os.pardir)) if 'GSC' in os.environ else os.getcwd()
    settings["commit"] = "LOCAL"
    settings["env"] = None
    settings["graal_distribution"] = "ce"

    parser = ArgumentParser(prog=sys.argv[0])
    parser.add_argument('--java-version', action='store', help='JDK version which should be downloaded (default: {})'.format(settings["java_version"]))
    parser.add_argument('--bootstrap-jdk', action='store', help='Bootstrap JDK location')
    parser.add_argument('--env', action='store', help='mx build environment')
    parser.add_argument('--install-deps', action='store_true', help='installs latest dependencies for your platform')
    parser.add_argument('--commit', action='store', help='(CAN BE DESTRUCTIVE!): LOCAL or HEAD or hash of the target commit (default: \'{}\')'.format(settings["commit"]))
    parser.add_argument('--clean', action='store_true', help='clean last build and tmp files (default: False)')
    parser.add_argument('--dev-mode', action='store', help='1 for fast build, 0 for AOT compiled build (default: {})'.format(settings["dev_mode"]))
    parser.add_argument('--dynamic-imports', '--dynamicimports', '--dy', action='store', help='mx dynamic imports (default: \'{}\')'.format(settings["dynamic_imports"]))
    parser.add_argument('--skip-libraries', action='store', help='comma separated list of libraries to skip, or "false" (default: \'{}\')'.format(settings["skip_libraries"]))
    parser.add_argument('--skip-fetch-jdk', action='store_true', help='skip fetching JDK - useful for local JDK builds (default: False)')
    parser.add_argument('--working-dir', action='store', help='working dir (default {})'.format(settings["working_dir"]))
    parser.add_argument('--graal-distribution', '--d', '-d', '--distro' , action='store', help='GraalVM distribution to build (default \'{}\')'.format(settings["graal_distribution"]))
    parser.add_argument('--version', action='store_true', help='shows script version')
    parser.add_argument('remainder', nargs=REMAINDER, metavar='...')
    args, unknownargs = parser.parse_known_args()
    args = vars(args)

    for name in args:
        if args[name] is not None:
            settings[name] = args[name]

    if settings["version"]:
        print(version)
        exit(0)

    check_update()

    if settings["install_deps"]:
        install_deps()

    old_working_dir = os.getcwd()
    os.chdir(settings["working_dir"])

    info("Getting mx build tools...")
    if not os.path.isdir('mx'):
        run('git clone --depth 1 https://github.com/graalvm/mx.git')
 
    my_env["MX_PYTHON_VERSION"] = '3'
    # my_env["MX_PYTHON"] = sys.executable

    mx_command = [os.path.join(settings["working_dir"], "mx", "mx"), "-y"]
    def mx(args):
        if isinstance(args, str):
            args = [args]
        run(mx_command + args)

    mx('update')

    info("Getting GraalVM sources...")
    if not os.path.isdir('graal'):
       run('git clone --depth 1 https://github.com/oracle/graal.git')

    if settings["commit"] != "LOCAL":
        os.chdir("graal")
        run("git checkout -- .")
        run("git fetch --all")
        run("git pull")
        run("git reset --hard " + settings["commit"])
        os.chdir(os.pardir)


    vm_path = os.path.join("graal", "vm") 
    if os.path.isdir('graal-enterprise'):
        print("    Detected GraalVM EE source code!")
        print("    You can enable it using --distro ee")
        if settings["graal_distribution"] == "ee":
            vm_path = os.path.join("graal-enterprise", "vm-enterprise")

    if not settings["skip_fetch_jdk"] and not "bootstrap_jdk" in settings:
        info("Getting right LabsJDK...")
        os.chdir(vm_path)
        jdk_distribution = settings["java_version"]
        try:
            java_num = int(jdk_distribution)
            if (java_num == 8):
                jdk_distribution = "openjdk8"
            elif (java_num >= 11):
                jdk_distribution = "labsjdk-{}-{}".format(settings["graal_distribution"], java_num)
        except:
            pass

        jdk_root = os.path.join(settings["working_dir"], "jdk", str(settings["java_version"]))
        if not os.path.isdir(jdk_root):
            os.makedirs(jdk_root)

        tmp_command = ["--no-warning", "fetch-jdk", "--to {}".format(jdk_root), "--java-distribution {}".format(jdk_distribution), "--alias latest"]

        if current_os == "darwin":
            tmp_command += ["--strip-contents-home"]
        mx(tmp_command)
        my_env["JAVA_HOME"] = os.path.join(jdk_root, "latest")
        os.chdir(settings["working_dir"])
    elif "bootstrap_jdk" in settings:
    	my_env["JAVA_HOME"] = settings["bootstrap_jdk"]

    os.chdir(settings["working_dir"])

    info("Preparing environment...")
    my_env["FASTR_CAPTURE_DEPENDENCIES"] = ""
    my_env["FASTR_RELEASE"] = "true"
    my_env["FASTR_NO_RECOMMENDED"] = "true"

    if platform == "linux":
        my_env["PKG_LDFLAGS_OVERRIDE"] = "-L/lib/x86_64-linux-gnu -L/usr/lib/x86_64-linux-gnu"
    elif platform == "darwin":
        my_env["PKG_INCLUDE_FLAGS_OVERRIDE"] = "-I/opt/local/include"
        my_env["PKG_LDFLAGS_OVERRIDE"] = "-L/opt/local/lib"

    if settings["clean"]:
        info("Cleaning untracked files")
        os.chdir("graal")
        run('git update-index -q --assume-unchanged truffle/external_repos/')
        run('git clean -qfdx')
        os.chdir(settings["working_dir"])

    info("Fetching dependencies and creating IDE configs...")
    os.chdir(vm_path)
    if settings["env"] is not None:
    	tmp_command = ["--env", settings["env"]]
    else:
    	tmp_command = ["--dynamicimports={}".format(settings["dynamic_imports"])]
    mx(tmp_command + ["sforceimports"])
    mx(tmp_command + ["--quiet", "intellijinit"])

    if settings["env"] is None:
        if settings["skip_libraries"] != 'false':
            tmp_command += ["--skip-libraries={}".format(settings["skip_libraries"])]
        tmp_command += ["--exclude-components={}".format(settings["exclude_components"])]

        if settings["dev_mode"] == '1':
            tmp_command += ["--force-bash-launchers=true", "--disable-installables=true"]
            my_env["LINKY_LAYOUT"] = "*.jar"

    # tmp_command += ["--J @-Xmx4g"]
    tmp_command += settings["remainder"]
    tmp_command += ["build"]

    info("Finally start building...")
    mx(tmp_command)
    os.chdir(os.path.join(settings["working_dir"], "graal", "vm"))
    graal_name = os.listdir(os.path.realpath("latest_graalvm"))[0]
    graal_location = os.path.join(os.path.realpath("latest_graalvm"), graal_name)

    os.chdir(settings["working_dir"])
    if not os.path.isdir('build'):
        os.mkdir('build')

    if settings["dev_mode"] == '1':
        graal_symlink = os.path.abspath(os.path.join("build", graal_name))
        if os.path.exists(graal_symlink) or os.path.islink(graal_symlink):
            rm(graal_symlink)
        os.symlink(graal_location, graal_symlink)
    else:
        new_dir = os.path.abspath(os.path.join("build", graal_name))
        if os.path.exists(new_dir):
            rm(new_dir)
        shutil.move(graal_location, "build")
        graal_location = new_dir
        if current_os != "windows":
            info("Compressing GraalVM distribution")
            os.chdir("build")
            run("tar -zcf  graalvm_latest.tar.gz {}".format(graal_name))
            os.chdir(settings["working_dir"])

    graal_symlink_latest = os.path.abspath(os.path.join("build", "latest"))
    if os.path.exists(graal_symlink_latest) or os.path.islink(graal_symlink_latest):
        rm(graal_symlink_latest)
    os.symlink(graal_location, graal_symlink_latest)

    os.chdir(old_working_dir)

    print()
    info("Add this to env variables:")
    print("------------------")
    print('GRAALVM_HOME=\"{}\"'.format(graal_symlink_latest))

    if settings["java_version"] == "11":
        print('JAVA11_HOME="{}"'.format(graal_symlink_latest))
    elif settings["java_version"] == "8":
        print('JAVA8_HOME="{}"'.format(graal_symlink_latest))

    print('PATH="$GRAALVM_HOME/bin:$PATH"')
    print('JAVA_HOME="$(dirname $(dirname $(realpath $(which java))))"')