WebKit remote build with delta downloads

webkit-remote-build.sh

#!/bin/bash
set -eu

BUILD_HOST=homura.local
BUILD_ARGS=(--gtk --debug)
REMOTE_BUILD_DIR=WebKitBuild/Debug
LOCAL_BUILD_DIR=WebKitBuild/Debug

# Files big enough to deserve xdelta transfer
# Generate with the following command:
# (cd WebKitBuild/Debug && find ./ -size +10000000c -exec ls -1 '{}' + |sed 's|^./||')
CHUBBY_FILES=(
bin/testb3
bin/TestWebKitAPI/WebCore/TestWebCore
bin/TestWebKitAPI/WTF/TestWTF
bin/WebKitPluginProcess2
build.ninja
compile_commands.json
DerivedSources/webkit2gtk/InspectorGResourceBundle.c
lib/libjavascriptcoregtk-4.0.so.18.7.0
lib/libTestRunnerInjectedBundle.so
lib/libWebCoreDerivedSources.a
lib/libWebCoreGTK.a
lib/libwebkit2gtk-4.0.so.37.25.0
.ninja_deps
)

if [ "$1" == "sync" ]; then
  # Generate a base snapshot of the chubby files in the build host and download
  # it to the local machine. It will be very slow on slow networks (over ~2 GB
  # download).
  ssh -T "$BUILD_HOST" <<END
set -eu
CHUBBY_FILES=($(for file in "${CHUBBY_FILES[@]}"; do printf "%s\n" "$file"; done))
mkdir -p "/webkit-remote-baseline/$REMOTE_BUILD_DIR"
cd "/webkit/$REMOTE_BUILD_DIR"
cp --parents "\${CHUBBY_FILES[@]}" "/webkit-remote-baseline/$REMOTE_BUILD_DIR"
END
  mkdir -p "/webkit-remote-baseline/$LOCAL_BUILD_DIR"
  rsync --info=progress2 -ax \
    "$BUILD_HOST:/webkit-remote-baseline/$REMOTE_BUILD_DIR/" \
    "/webkit-remote-baseline/$LOCAL_BUILD_DIR/"

elif [ "$1" == "check" ]; then
  # Test that the baseline chubby files match. Otherwise the delta files will
  # not be applied correctly.
  (cd "/webkit-remote-baseline/$LOCAL_BUILD_DIR" && md5sum "${CHUBBY_FILES[@]}" > /tmp/baseline-local.txt)
  ssh -T "$BUILD_HOST" <<END > /tmp/baseline-remote.txt
cd "/webkit-remote-baseline/$REMOTE_BUILD_DIR" && md5sum ${CHUBBY_FILES[@]}
END
  diff -u /tmp/baseline-{local,remote}.txt
  if [ $? -eq 0 ]; then
    echo "Baseline is correctly synchronized between the local machine and the build machine."
    exit 0
  else
    exit 1
  fi

elif [ "$1" == "build" ]; then
  cd /webkit
  commit_hash="$(git show --format="%h" --no-patch)"
  git diff --cached > /tmp/local-changes.patch

  rsync /tmp/local-changes.patch "$BUILD_HOST":/tmp/
  ssh -T "$BUILD_HOST" <<END
set -eu
cd /webkit

# Fetch new commits if necessary
if ! git show "$commit_hash" >/dev/null 2>/dev/null; then
  git fetch --all
fi

# Delete old changes
git clean -fd
git checkout -- . >/dev/null

# Put the source tree in the same state as the client
git checkout "$commit_hash"
patch -p1 < /tmp/local-changes.patch
./Tools/Scripts/build-webkit ${BUILD_ARGS[@]}

# Generate deltas
echo "Generating delta files..."
time_start_deltas=$SECONDS

rm -rf "/webkit-remote-deltas/$REMOTE_BUILD_DIR"
for file in ${CHUBBY_FILES[@]}; do
  printf "Generating delta: %s\n" "\$file"
  mkdir -p "/webkit-remote-deltas/$REMOTE_BUILD_DIR/\$(dirname "\$file")"
  xdelta -0 -N -e -s /webkit-remote-baseline/$REMOTE_BUILD_DIR/\$file \
    $REMOTE_BUILD_DIR/\$file \
    /webkit-remote-deltas/$REMOTE_BUILD_DIR/\$file.delta
done

total_time_deltas=\$(date -u -d \
  "0 \$SECONDS seconds - \$time_start_deltas seconds" \
  +"%H:%M:%S")
printf "All deltas generated in %s" "\$total_time_deltas"

# Print the size of the generated deltas
find /webkit-remote-deltas -type f -exec du -h {} + | sort -h
END

  time_start_transfer=$SECONDS
  echo "Transferring whole files..."
  mkdir -p "/webkit/$LOCAL_BUILD_DIR/"
  rsync --delete --info=progress2 -aXz \
    --exclude='*.o' --exclude='*.dwo' \
    $(for file in "${CHUBBY_FILES[@]}"; do printf "%s=%s " --exclude "/$file"; done) \
    "$BUILD_HOST:/webkit/$REMOTE_BUILD_DIR/" "/webkit/$LOCAL_BUILD_DIR/"

  echo "Transferring delta files..."
  mkdir -p "/webkit-remote-deltas/$LOCAL_BUILD_DIR/"
  rsync --delete --info=progress2 -aX \
    "$BUILD_HOST:/webkit-remote-deltas/$REMOTE_BUILD_DIR/" \
    "/webkit-remote-deltas/$LOCAL_BUILD_DIR/"

  echo "Applying deltas..."
  for file in "${CHUBBY_FILES[@]}"; do
    printf "Applying delta: %s\n" "$file"
    xdelta -d -s "/webkit-remote-baseline/$LOCAL_BUILD_DIR/$file" \
      "/webkit-remote-deltas/$LOCAL_BUILD_DIR/$file.delta" \
      "$LOCAL_BUILD_DIR/$file.delta"
  done

  total_time_transfer=$(date -u -d \
    "0 $SECONDS seconds - $time_start_transfer seconds" \
    +"%H:%M:%S")
  printf "Remote build fetched in %s" "$total_time_transfer"

else
  echo "Unknown command"
  exit 1
fi