Script for building NW.js for an ARM device.

build_nwjs_arm.sh

#!/usr/bin/env bash

set -e

FIRST_RUN=1
BUILD_ARM=1
BUILD_SDK=1
BUILD_NACL=1
NWJS_BRANCH="nw44"

export LC_ALL=C.UTF-8


# Steps and arguments taken from:
#   http://buildbot-master.nwjs.io:8010/builders/nw44_sdk_linux64/builds/27
#     Click on a step's logs to see the environment variables/commands
#   https://github.com/LeonardLaszlo/nw.js-armv7-binaries/blob/master/building-script.sh
#   https://github.com/LeonardLaszlo/nw.js-armv7-binaries/blob/master/docs/build-nwjs-v0.28.x.md

# For defines/args references, see:
#   https://web.archive.org/web/20160818205525/https://chromium.googlesource.com/chromium/src/+/master/tools/gn/docs/cookbook.md
#   https://github.com/nwjs/chromium.src/blob/22851eb48fe67f4419acc9b61b45ce61aad2c90b/native_client_sdk/src/build_tools/buildbot_run.py#L26

export GYP_CHROMIUM_NO_ACTION=0
export GYP_DEFINES="building_nw=1 buildtype=Official clang=1 OS=linux"
export GN_ARGS="is_debug=false target_os=\"linux\" is_component_ffmpeg=true is_component_build=false symbol_level=1 ffmpeg_branding=\"Chrome\""

if (( BUILD_SDK )); then
    export GYP_DEFINES="${GYP_DEFINES} nwjs_sdk=1"
    export GN_ARGS="${GN_ARGS} nwjs_sdk=true"
else
    export GYP_DEFINES="${GYP_DEFINES} nwjs_sdk=0"
    export GN_ARGS="${GN_ARGS} nwjs_sdk=false"
fi

if (( BUILD_NACL )); then
    export GYP_DEFINES="${GYP_DEFINES} disable_nacl=0"
    export GN_ARGS="${GN_ARGS} enable_nacl=true"
else
    export GYP_DEFINES="${GYP_DEFINES} disable_nacl=1"
    export GN_ARGS="${GN_ARGS} enable_nacl=false"
fi

if (( BUILD_ARM )); then
    export GYP_DEFINES="${GYP_DEFINES} target_arch=arm target_cpu=arm arm_float_abi=hard"
    export GN_ARGS="${GN_ARGS} target_cpu=\"arm\" arm_float_abi=\"hard\""
else
    export GYP_DEFINES="${GYP_DEFINES} target_arch=x64"
    export GN_ARGS="${GN_ARGS} target_cpu=\"x64\""
fi



################################ Initial Setup #################################

# Pre-requisites
if (( FIRST_RUN )); then
    sudo apt-get install -y git python file lsb-release
fi


# Cloning Chromium depot tools.
if [[ ! -d "depot_tools" ]]; then
    git clone "https://chromium.googlesource.com/chromium/tools/depot_tools.git"
fi
export PATH="${PATH}:${PWD}/depot_tools"


# Configuring gclient for NW.js.
cat <<CONFIG > ".gclient"
solutions = [
  { "name"        : 'src',
    "url"         : 'https://github.com/nwjs/chromium.src.git@origin/${NWJS_BRANCH}',
    "deps_file"   : 'DEPS',
    "managed"     : True,
    "custom_deps" : {
        "src/third_party/WebKit/LayoutTests": None,
        "src/chrome_frame/tools/test/reference_build/chrome": None,
        "src/chrome_frame/tools/test/reference_build/chrome_win": None,
        "src/chrome/tools/test/reference_build/chrome": None,
        "src/chrome/tools/test/reference_build/chrome_linux": None,
        "src/chrome/tools/test/reference_build/chrome_mac": None,
        "src/chrome/tools/test/reference_build/chrome_win": None,
    },
    "custom_vars": {},
  },
]
CONFIG


clone_or_fetch() {
    repo_url="${1}"
    repo_path="${2}"
    repo_branch="${3}"
    if [[ -d "${repo_path}" ]]; then
        (
            cd "${repo_path}" &&
            git fetch --tags "${repo_url}" "${repo_branch}" &&
            git reset --hard origin/"${repo_branch}"
        )
    else
        git clone --depth 1 --branch "${repo_branch}" "${repo_url}" "${repo_path}"
    fi
}

# Cloning sources to required paths.
clone_or_fetch "https://github.com/nwjs/nw.js" "src/content/nw"          "${NWJS_BRANCH}"
clone_or_fetch "https://github.com/nwjs/node"  "src/third_party/node-nw" "${NWJS_BRANCH}"
clone_or_fetch "https://github.com/nwjs/v8"    "src/v8"                  "${NWJS_BRANCH}"


# Clone Chromium (and go grab a big cup of coffee).
sync_args=(--with_branch_heads --reset --verbose)
if (( FIRST_RUN )); then
    sync_args+=(--nohooks)
fi
gclient sync "${sync_args[@]}"


cd src


# Installing build dependencies (and go grab a small cup of coffee).
build_deps_args=(--no-prompt --no-backwards-compatible)
if (( BUILD_ARM )); then
    # Although allegedly enabled by default, we should be explicit just in case.
    build_deps_args+=(--arm)
fi
if (( FIRST_RUN )); then
    sudo ./build/install-build-deps.sh "${build_deps_args[@]}"
fi

if (( BUILD_ARM )); then
    # Installing required sysroot files for ARM target architecture.
    # WARNING: don't sudo this command or you'll be in for a bad time.
    #   It will cause the sysroot to extract with a different owner/group ID
    #   that non-root users won't be able to read.  This silently causes
    #   part of the `gn gen` step to fail and you will waste hours
    #   trying to track down the reason.
    build/linux/sysroot_scripts/install-sysroot.py --arch=arm
fi

# Pull all other required dependencies.
if (( FIRST_RUN )); then
    gclient runhooks
fi



################################### Patches ####################################

# For nwjs_sdk=false builds, some required(?) files never get built.
# As a workaround, always use the SDK's GRIT input regardless of the flag.
#   See: https://github.com/nwjs/chromium.src/issues/145
git am <<'PATCH' || git am --abort
From dc3860edac430b1635050141343f6b6b34b1c451 Mon Sep 17 00:00:00 2001
From: llamasoft <llamasoft@rm-rf.email>
Date: Thu, 20 Feb 2020 18:17:06 -0500
Subject: [PATCH] Always use SDK GRIT input file

---
 chrome/browser/BUILD.gn | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/chrome/browser/BUILD.gn b/chrome/browser/BUILD.gn
index 3f15d567deca..3d1a1db7cc31 100644
--- a/chrome/browser/BUILD.gn
+++ b/chrome/browser/BUILD.gn
@@ -5373,11 +5373,7 @@ proto_library("permissions_proto") {
 }

 grit("resources") {
-  if (nwjs_sdk) {
-    source = "browser_resources.grd"
-  } else {
-    source = "nwjs_resources.grd"
-  }
+  source = "browser_resources.grd"

   # The .grd contains references to generated files.
   source_is_generated = true
--
2.17.1
PATCH



############################ Build File Generation #############################

# Patching third_party/node-nw/common.gypi is no longer required!  Thank you Roger!
#   See: https://github.com/nwjs/node/pull/38

# Modifying package_binaries.py is no longer required!  Thank you Roger!
#   See: https://github.com/nwjs/nw.js/pull/7382

# Generating build files for NW.js
gn gen "out/nw" --args="${GN_ARGS}"

# Generating build files for node.
./build/gyp_chromium -I third_party/node-nw/common.gypi third_party/node-nw/node.gyp



############################# Project Compilation ##############################

# Build the beast (and go grab a lunch, another coffee, and a snack).
# On a clean build, this step takes around 5 hours.
ninja -C out/nw nwjs

# This outputs a few warnings.  That's probably ok?
ninja -C out/Release node

# Copying GYP built node to GN output directory.
ninja -C out/nw copy_node

# `content/nw/BUILD.gn` uses the `build/linux/dump_app_syms.py` tool to strip binaries.
# This is an issue, as the `dump_app_syms.py` script calls the host's `strip` executable
# which is unlikely to support stripping the ARM architecture binaries.
# As a workaround, we create a wrapper `strip` script that uses `llvm-objcopy`
# from Chromium's build toolchain which supports almost every architecture.
if (( BUILD_ARM )); then
    temp_dir=$(mktemp -d)
    OLD_PATH="${PATH}"
    export PATH="${temp_dir}:${PATH}"

    # Typically under `third_party/llvm-build/Release+Asserts/bin`, but search for it just in case.
    objcopy=$(find . -type f -name "llvm-objcopy" | head -1 | xargs -n 1 realpath)
    cat > "${temp_dir}/strip" <<STRIP_SCRIPT 
#!/bin/sh
"${objcopy}" --strip-unneeded "\$@"
STRIP_SCRIPT
    chmod +x "${temp_dir}/strip"
fi

# Extracting symbols and stripping binaries.
ninja -C out/nw dump

if (( BUILD_ARM )); then
    export PATH="${OLD_PATH}"
    rm -rf "${temp_dir}"
fi

# Packaging results for distribution.
# Results end up in out/nw/dist.
ninja -C out/nw dist

If I try to build nw44 with no sdk and no nacl and with ffmpeg branding it fails with:

[23161/37551] ACTION //chrome/browser:resources_grit(//build/toolchain/linux:clang_x64)
FAILED: gen/chrome/resources_stamp.d.stamp gen/chrome/grit/browser_resources.h gen/chrome/browser_resources.pak gen/chrome/browser_resources.pak.info 
python ../../tools/grit/grit.py -i ../../chrome/browser/nwjs_resources.grd build -o gen/chrome --depdir . --depfile gen/chrome/resources_stamp.d --write-only-new=1 --depend-on-stamp -D scale_factors=2x -D _chromium -E CHROMIUM_BUILD=chromium -D desktop_linux -D toolkit_views -D use_aura -D use_nss_certs --brotli brotli -D enable_arcore=false -D enable_background_mode=true -D enable_background_contents=true -D enable_extensions=true -D enable_hangout_services_extension=false -D enable_plugins=true -D enable_print_preview=true -D enable_printing=true -D enable_service_discovery=true -D enable_supervised_users=false -D enable_vr=true -D enable_webui_tab_strip=false -D safe_browsing_mode=0 -D optimize_webui=false -E additional_modules_list_file=gen/chrome/browser/internal/additional_modules_list.txt -E root_gen_dir=gen -f ../../tools/gritsettings/resource_ids --assert-file-list=obj/chrome/browser/resources_expected_outputs.txt
parse exception: run GRIT with the -x flag to debug .grd problems
Traceback (most recent call last):
  File "../../tools/grit/grit.py", line 23, in <module>
    sys.exit(grit.grit_runner.Main(sys.argv[1:]))
  File "/home/docker/nwjs/src/tools/grit/grit/grit_runner.py", line 310, in Main
    return toolobject.Run(options, args[1:])
  File "/home/docker/nwjs/src/tools/grit/grit/tool/build.py", line 257, in Run
    target_platform=target_platform)
  File "/home/docker/nwjs/src/tools/grit/grit/grd_reader.py", line 202, in Parse
    xml.sax.parse(filename_or_stream, handler)
  File "/usr/lib/python2.7/xml/sax/__init__.py", line 33, in parse
    parser.parse(source)
  File "/usr/lib/python2.7/xml/sax/expatreader.py", line 110, in parse
    xmlreader.IncrementalParser.parse(self, source)
  File "/usr/lib/python2.7/xml/sax/xmlreader.py", line 123, in parse
    self.feed(buffer)
  File "/usr/lib/python2.7/xml/sax/expatreader.py", line 213, in feed
    self._parser.Parse(data, isFinal)
  File "/usr/lib/python2.7/xml/sax/expatreader.py", line 320, in end_element
    self._cont_handler.endElement(name)
  File "/home/docker/nwjs/src/tools/grit/grit/grd_reader.py", line 105, in endElement
    self.stack.pop().EndParsing()
  File "/home/docker/nwjs/src/tools/grit/grit/node/misc.py", line 355, in EndParsing
    self.ValidateUniqueIds()
  File "/home/docker/nwjs/src/tools/grit/grit/node/misc.py", line 388, in ValidateUniqueIds
    raise exception.DuplicateKey(', '.join(duplicate_names))
grit.exception.DuplicateKey: A duplicate key attribute was found.: IDR_MD_DOWNLOADS_ITEM_HTML, IDR_MD_DOWNLOADS_ITEM_JS, IDR_MD_DOWNLOADS_MANAGER_HTML, IDR_MD_DOWNLOADS_MANAGER_JS, IDR_MD_DOWNLOADS_TOOLBAR_HTML, IDR_MD_DOWNLOADS_TOOLBAR_JS, IDR_DOWNLOADS_MANAGER_JS
[23166/37551] CXX obj/url/mojom/url_mojom_origin_blink/origin.mojom-blink.o
ninja: build stopped: subcommand failed.

Odd, I didn't run into that issue. Then again, I didn't try to build no-sdk no-NaCl. I'll fire up an environment and see if I can reproduce the issue. I'm not sure if it's relevant, but I was building on Ubuntu 18.04.4 LTS.
In the meantime, it's probably worth digging into what generated chrome/browser/nwjs_resources.grd. It's going to take me a bit to get another build environment created.

If you need a build environment you can use this one: https://hub.docker.com/repository/docker/laslaul/nwjs-arm-build-env

I reran the build last night with no-SDK and no-NaCl and it ran successfully. Here are the flags I used:

export GYP_CHROMIUM_NO_ACTION=0
export GYP_DEFINES="building_nw=1 nwjs_sdk=0 disable_nacl=1 buildtype=Official clang=1 OS=linux"
export GN_ARGS="is_debug=false target_os=\"linux\" is_component_ffmpeg=true symbol_level=1 enable_nacl=false ffmpeg_branding=\"Chrome\""
if (( BUILD_ARM )); then
    export GYP_DEFINES="${GYP_DEFINES} target_arch=arm target_cpu=arm arm_float_abi=hard"
    export GN_ARGS="${GN_ARGS} target_cpu=\"arm\" arm_float_abi=\"hard\""
else
    export GYP_DEFINES="${GYP_DEFINES} target_arch=x64"
    export GN_ARGS="${GN_ARGS} target_cpu=\"x64\""
fi

I've have a look at your Docker image to see if I can replicate your issue. I've been running on a VM, so hopefully that doesn't make too much a difference.

Can you send me the contents of the /usr/src/nwjs/build-container.sh script that runs in your Docker image? I'd like to avoid downloading a 20GB image if possible. 😁

@LeonardLaszlo Sorry for spamming, but I updated the build script. It looks like the nwjs_sdk flag was present in GYP_DEFINES, but not GN_ARGS. One known side-effect is that it will cause the dist step to fail. I'm not sure if the changes will fix your particular issue, but it's worth a shot.

I'm not getting the exact error you're getting, but I am encountering a failure when building the chrome/browser:resources_grit target:

/data/nwjs/src$ time ninja -C out/nw 'chrome/browser:resources_grit'
ninja: Entering directory `out/nw'
[556/557] ACTION //chrome/browser:resources_grit(//build/toolchain/linux:clang_x64)
FAILED: gen/chrome/resources_stamp.d.stamp gen/chrome/grit/browser_resources.h gen/chrome/browser_resources.pak gen/chrome/browser_resources.pak.info
python ../../tools/grit/grit.py -i ../../chrome/browser/nwjs_resources.grd build -o gen/chrome --depdir . --depfile gen/chrome/resources_stamp.d --write-only-new=1 --depend-on-stamp -D scale_factors=2x -D _chromium -E CHROMIUM_BUILD=chromium -D desktop_linux -D toolkit_views -D use_aura -D use_nss_certs --brotli brotli -D enable_arcore=false -D enable_background_mode=true -D enable_background_contents=true -D enable_extensions=true -D enable_hangout_services_extension=false -D enable_plugins=true -D enable_print_preview=true -D enable_printing=true -D enable_service_discovery=true -D enable_supervised_users=false -D enable_vr=true -D enable_webui_tab_strip=false -D safe_browsing_mode=0 -D optimize_webui=true -E additional_modules_list_file=gen/chrome/browser/internal/additional_modules_list.txt -E root_gen_dir=gen -f ../../tools/gritsettings/resource_ids --assert-file-list=obj/chrome/browser/resources_expected_outputs.txt
Error processing node <?xml version="1.0" encoding="UTF-8"?>
<include allowexternalscript="true" compress="gzip" file="${root_gen_dir}\chrome\browser\resources\discards\graph_tab.html" name="IDR_DISCARDS_GRAPH_TAB_HTML" type="BINDATA" use_base_dir="false" />: [Errno 2] No such file or directory: u'../../out/nw/gen/chrome/browser/resources/discards/graph_tab.html'
Traceback (most recent call last):
  File "../../tools/grit/grit.py", line 23, in <module>
    sys.exit(grit.grit_runner.Main(sys.argv[1:]))
  File "/data/nwjs/src/tools/grit/grit/grit_runner.py", line 310, in Main
    return toolobject.Run(options, args[1:])
  File "/data/nwjs/src/tools/grit/grit/tool/build.py", line 272, in Run
    self.Process()
  File "/data/nwjs/src/tools/grit/grit/tool/build.py", line 404, in Process
    self.ProcessNode(self.res, output, outfile)
  File "/data/nwjs/src/tools/grit/grit/tool/build.py", line 331, in ProcessNode
    formatted = formatter(node, output_node.GetLanguage(), output_dir=base_dir)
  File "/data/nwjs/src/tools/grit/grit/format/data_pack.py", line 88, in Format
    value = node.GetDataPackValue(lang, UTF8)
  File "/data/nwjs/src/tools/grit/grit/node/include.py", line 107, in GetDataPackValue
    data = util.ReadFile(filename, util.BINARY)
  File "/data/nwjs/src/tools/grit/grit/util.py", line 210, in ReadFile
    with open(filename, mode) as f:
IOError: [Errno 2] No such file or directory: u'../../out/nw/gen/chrome/browser/resources/discards/graph_tab.html'
ninja: build stopped: subcommand failed.

This appears to be triggered directly by the nwjs_sdk=false flag, not the NaCl flags. I also tried doing a clean build using the parameters defined in this buildbot job and encountered the same error.

From what I can tell, when updating to Chromium 80.0, the graph_tab target was removed from chrome/browser/resources/discards/BUILD.gn.

I think this is a non-issue for nwjs_sdk=true builds because an entirely different resource pack is used. This is from chrome/browser/BUILD.gn:

grit("resources") {
  if (nwjs_sdk) {
    source = "browser_resources.grd"
  } else {
    source = "nwjs_resources.grd"
  }

Given that you're also running into grit-related issues, I think this is the likely cause.
I've submitted an issue for this and I'm looking into a workaround for it. Hopefully just removing the above conditional and defaulting to browser_resources.grd will let this run to completion.

You can find my scripts in the root of: https://github.com/LeonardLaszlo/nw.js-armv7-binaries

My docker images apparently work only on mac os.

That's odd. I did make a tweak to the build script last night that patches chrome/browser/BUILD.gn. Did the MacOS run include that change?

Will this work on Raspberry Pi 1/Zero? They use ARMv6 instead of ARMv7...

@jalbam No. The current script builds to ARMv7 because that's what Chromium considers the "ARM architecture". Even if you could, I don't think the Raspberry Pi Zero has the memory or processing power to effectively run any NW.js app.

Btw I managed to build all flavours of nwjs, but it is odd to see that no sdk and no nacl versions have almost the same size as sdk and nacl versions.

I hope I'll have the time to test it next week.

@llamasoft the reason why your build didn't fail is because your GN_ARGS lack the nwjs_sdk=false flag. All the builds that succeeded for your are sdk flavors of nwjs.

I am getitng the same error when building with nwjs_sdk=false flag.

Done. Made 13760 targets from 1998 files in 14627ms
Updating projects from gyp files...
ninja: Entering directory `out/nw'
[14627/57448] ACTION //chrome/browser:resources_grit(//build/toolchain/linux:clang_arm)
FAILED: gen/chrome/resources_stamp.d.stamp gen/chrome/grit/browser_resources.h gen/chrome/browser_resources.pak gen/chrome/browser_resources.pak.info 
python ../../tools/grit/grit.py -i ../../chrome/browser/nwjs_resources.grd build -o gen/chrome --depdir . --depfile gen/chrome/resources_stamp.d --write-only-new=1 --depend-on-stamp -D scale_factors=2x -D _chromium -E CHROMIUM_BUILD=chromium -D desktop_linux -D toolkit_views -D use_aura -D use_nss_certs -t linux2 --brotli clang_x64/brotli -D enable_arcore=false -D enable_background_mode=true -D enable_background_contents=true -D enable_extensions=true -D enable_hangout_services_extension=false -D enable_plugins=true -D enable_print_preview=true -D enable_printing=true -D enable_service_discovery=true -D enable_supervised_users=false -D enable_vr=false -D enable_webui_tab_strip=false -D safe_browsing_mode=0 -D optimize_webui=true -E additional_modules_list_file=gen/chrome/browser/internal/additional_modules_list.txt -E root_gen_dir=gen -f ../../tools/gritsettings/resource_ids --assert-file-list=obj/chrome/browser/resources_expected_outputs.txt
Error processing node <?xml version="1.0" encoding="UTF-8"?>
<include allowexternalscript="true" compress="gzip" file="${root_gen_dir}\chrome\browser\resources\discards\graph_tab.html" name="IDR_DISCARDS_GRAPH_TAB_HTML" type="BINDATA" use_base_dir="false" />: [Errno 2] No such file or directory: u'../../out/nw/gen/chrome/browser/resources/discards/graph_tab.html'
Traceback (most recent call last):
  File "../../tools/grit/grit.py", line 23, in <module>
    sys.exit(grit.grit_runner.Main(sys.argv[1:]))
  File "/usr/docker/nwjs/src/tools/grit/grit/grit_runner.py", line 310, in Main
    return toolobject.Run(options, args[1:])
  File "/usr/docker/nwjs/src/tools/grit/grit/tool/build.py", line 272, in Run
    self.Process()
  File "/usr/docker/nwjs/src/tools/grit/grit/tool/build.py", line 404, in Process
    self.ProcessNode(self.res, output, outfile)
  File "/usr/docker/nwjs/src/tools/grit/grit/tool/build.py", line 331, in ProcessNode
    formatted = formatter(node, output_node.GetLanguage(), output_dir=base_dir)
  File "/usr/docker/nwjs/src/tools/grit/grit/format/data_pack.py", line 88, in Format
    value = node.GetDataPackValue(lang, UTF8)
  File "/usr/docker/nwjs/src/tools/grit/grit/node/include.py", line 107, in GetDataPackValue
    data = util.ReadFile(filename, util.BINARY)
  File "/usr/docker/nwjs/src/tools/grit/grit/util.py", line 210, in ReadFile
    with open(filename, mode) as f:
IOError: [Errno 2] No such file or directory: u'../../out/nw/gen/chrome/browser/resources/discards/graph_tab.html'
[14632/57448] CXX obj/components/safe_browsing/csd_proto/csd.pb.o
ninja: build stopped: subcommand failed.