Skip to content

Instantly share code, notes, and snippets.

@j1elo
Created January 13, 2023 02:20
Show Gist options
  • Save j1elo/6156437f26bfc580769613bb30419af8 to your computer and use it in GitHub Desktop.
Save j1elo/6156437f26bfc580769613bb30419af8 to your computer and use it in GitHub Desktop.
Monorepo merger for the Kurento project
#!/usr/bin/env bash
# Checked with ShellCheck (https://www.shellcheck.net/)
# Shell setup
# ===========
# Bash options for strict error checking.
set -o errexit -o errtrace -o pipefail -o nounset
shopt -s inherit_errexit 2>/dev/null || true
# Trace all commands (to stderr).
#set -o xtrace
# Merge functions
# ===============
# Merge function based on the method used by GStreamer for their monorepo.
#
# This worked acceptably but it has several shortcomings, such as difficulties
# when having a collision between old and new paths. Otherwise, a good thing is
# that the preparation steps are kept in the monorepo itself.
#
# Sources:
# * https://blogs.gnome.org/tsaunier/2021/09/29/gstreamer-one-repository-to-rule-them-all/
# - Archive: https://web.archive.org/web/20221226070556/https://blogs.gnome.org/tsaunier/2021/09/29/gstreamer-one-repository-to-rule-them-all/
# * https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/474
#
# This is mostly a direct translation of the procedure from source code:
# https://gitlab.freedesktop.org/thiblahute/gst-merger/blob/master/merge.py
function merge_repo_gstreamer {
if [[ $# -ne 4 ]]; then
echo "Expected arguments: <Name> <URL> <Branch> <DestDir>"
return 1
fi
MODULE_NAME="$1"
MODULE_URL="$2"
MODULE_BRANCH="$3"
MODULE_DIR="$4"
git remote add "$MODULE_NAME" "$MODULE_URL"
git remote update "$MODULE_NAME"
git merge --no-edit --allow-unrelated-histories --message "Merge '$MODULE_NAME' into monorepo" "$MODULE_NAME/$MODULE_BRANCH"
mapfile -t FILES < <(git diff --name-only 'HEAD^..HEAD')
for FILE in "${FILES[@]}"; do
FILE_SRC="$(dirname "$FILE")"
FILE_DST="$MODULE_DIR/$FILE_SRC"
mkdir --parents "$FILE_DST"
git mv "$FILE" "$FILE_DST"
done
git commit --no-edit --no-verify --message "Move files from '$MODULE_NAME' into $MODULE_DIR/"
git clean -xdf
git remote remove "$MODULE_NAME"
}
# Merge function based on the method shown by the JDriven blog.
#
# Doing monorepo preparation separately on the source repository is much less
# likely to hit a name collision between old and new paths.
#
# This function implements the original idea, and multiple safeguards to keep
# an eye on corner cases such as spaces in filenames, hidden destination
# dirs, and git submodules.
#
# Sources:
# * https://blog.jdriven.com/2021/04/how-to-merge-multiple-git-repositories/
# - Archive: https://web.archive.org/web/20210518082201/https://blog.jdriven.com/2021/04/how-to-merge-multiple-git-repositories/
function merge_repo_jdriven {
if [[ $# -ne 4 ]]; then
echo "Expected arguments: <Name> <URL> <Branch> <DestDir>"
return 1
fi
MODULE_NAME="$1"
MODULE_URL="$2"
MODULE_BRANCH="$3"
MODULE_DIR="$4"
git clone --branch "$MODULE_BRANCH" "$MODULE_URL" "/tmp/$MODULE_NAME"
pushd "/tmp/$MODULE_NAME"
{
git switch --create monorepo-preparation
# Create a unique temporary working directory. This avoids the chance of
# collision between the <DestDir> and a possibly already existing dir.
# For example, "kurento-media-server" already contains a "server" subdir
# which collides with the "server" destination in monorepo.
TEMP_DIR="$(mktemp --directory --tmpdir=.)"
DEST_DIR="$TEMP_DIR/$MODULE_DIR"
mkdir --parents "$DEST_DIR"
# Specific per-repo fixes.
if [[ "$MODULE_NAME" == "kms-omni-build" ]]; then
# Delete all submodules, because they will be individually added
# later under their new locations.
mapfile -t SUBMODULES < <(git submodule status | awk '{print $2}')
git rm "${SUBMODULES[@]}"
git rm .gitmodules
git clean --force .
git commit --no-verify --message "dummy"
fi
# Make an array with the names of all top-level git entries.
mapfile -t FILES < <(git ls-tree --name-only HEAD)
# Exclude files that git requires to keep in the repo's root.
FILES_AUX=()
for FILE in "${FILES[@]}"; do
if [[ "$FILE" != ".gitmodules" ]]; then
FILES_AUX+=("$FILE")
fi
done
FILES=("${FILES_AUX[@]}")
unset FILES_AUX
# Move all git entries to their destinations.
git mv "${FILES[@]}" "$DEST_DIR"
# Now move all files out of the temp dir.
# `.[!.]*` is to avoid including `.` and `..` in the wildcard for hidden
# files. It would also exclude things like `..file`, but that's a very
# uncommon file name anyways.
git mv "$TEMP_DIR"/* ./ || true
git mv "$TEMP_DIR"/.[!.]* ./ || true
rmdir "$TEMP_DIR"
# `--amend` used here in case a previous dummy commit was done.
git commit --amend --no-verify \
--message "monorepo: move files into $MODULE_DIR/"
# Push this preparation to the remote, so we leave a traceable branch.
# WARNING: ONLY DO THIS IN THE DEFINITIVE RUN OF THE SCRIPT.
# LEAVE COMMENTED OTHERWISE.
#git push --set-upstream origin HEAD
}
popd
git remote add "$MODULE_NAME" "/tmp/$MODULE_NAME"
{
git fetch "$MODULE_NAME"
git merge \
--allow-unrelated-histories \
--message "monorepo: merge '$MODULE_NAME' into $MODULE_DIR/" \
"$MODULE_NAME/monorepo-preparation"
# Specific per-repo fixes.
if [[ "$MODULE_NAME" == "kurento-java" ]]; then
# Resolve the conflict caused by a past removal of maven-plugin.
git restore --staged --worktree clients/java/maven-plugin/
git add . && git commit --no-edit
fi
}
git remote remove "$MODULE_NAME"
}
# Merge function based on the "Subtree Merging" technique.
#
# This seems to be the best monorepo merge technique for the needs of a Kurento
# monorepo. It is much simpler than the other methods, while keeping nice
# properties such as being safe against name collisions (which the GStreamer
# way doesn't prevent) and also keeping the whole migration in the commit
# history of the monorepo itself (which the JDriven method lacks).
#
# This way of merging requires, however, manually adjusting the git submodules
# file (.gitmodules) that each subproject might bring to the monorepo.
#
# Sources:
# * https://mirrors.edge.kernel.org/pub/software/scm/git/docs/howto/using-merge-subtree.html
# * https://docs.github.com/en/get-started/using-git/about-git-subtree-merges
# * https://stackoverflow.com/questions/1425892/how-do-you-merge-two-git-repositories
function merge_repo_subtree {
if [[ $# -ne 4 ]]; then
echo "Expected arguments: <Name> <URL> <Branch> <DestDir>"
return 1
fi
MODULE_NAME="$1"
MODULE_URL="$2"
MODULE_BRANCH="$3"
MODULE_DIR="$4"
if [[ "$MODULE_DIR" -ef . ]]; then
echo "ERROR: <DestDir> must be a subdirectory in the monorepo"
exit 1
fi
# Add the source URL as a git remote. This simplifies the next commands.
git remote add "$MODULE_NAME" "$MODULE_URL"
# Fetch only commits.
# Uses the default refspec, which is "refs/remotes/<RemoteName>/". This way,
# when the remote is removed, its unmerged branches are removed too.
git fetch --no-tags "$MODULE_NAME"
# Fetch only tags.
# Use an explicit refspec, in order to save tags under the given prefix.
git fetch --no-tags "$MODULE_NAME" "+refs/tags/*:refs/tags/$MODULE_NAME/*"
# Create a merge commit, but leave it empty ("ours" strategy).
# The actual addition of files will be done next.
git merge \
--strategy=ours \
--no-commit \
--allow-unrelated-histories \
"$MODULE_NAME/$MODULE_BRANCH"
# Add files from the given branch, under the given prefix.
# Because we are in the middle of an unfinished merge, this will show up as
# files added by the merge commit.
git read-tree -u --prefix="$MODULE_DIR" "$MODULE_NAME/$MODULE_BRANCH"
# Specific per-repo fixes.
if [[ "$MODULE_NAME" == "kms-omni-build" ]]; then
# Delete all submodules, because they will be individually added
# later under their new locations.
mapfile -t SUBMODULES < <(sed --quiet --regexp-extended "s|^.*path = (.*)|$MODULE_DIR/\1|p" "$MODULE_DIR/.gitmodules")
git rm --force "${SUBMODULES[@]}"
git rm --force "$MODULE_DIR/.gitmodules"
git clean --force .
fi
# If the repo comes with a `.gitmodules`, integrate it in the monorepo.
if [[ -f "$MODULE_DIR/.gitmodules" ]]; then
sed --regexp-extended \
--expression "s|submodule \"(.*)\"|submodule \"$MODULE_DIR/\1\"|" \
--expression "s|path = (.*)|path = $MODULE_DIR/\1|" \
"$MODULE_DIR/.gitmodules" \
>>.gitmodules
git rm --force "$MODULE_DIR/.gitmodules"
git add .gitmodules
fi
# Commit changes, finishing the ongoing merge.
git commit --message "monorepo: merge '$MODULE_NAME' into $MODULE_DIR/"
# Remove the git remote, which also removes all of its unmerged branches.
# This leaves a clean repo graph.
git remote remove "$MODULE_NAME"
}
# Entrypoint functions
# ====================
# NOT TESTED: I went directly to the other two methods, they seem better to me.
function prepare_merge_gstreamer {
mkdir monorepo-gstreamer
cd monorepo-gstreamer/
git init --initial-branch "main" .
git commit --allow-empty --message "monorepo: Initial commit"
function merge_repo {
merge_repo_gstreamer "$@"
}
}
function prepare_merge_jdriven {
mkdir monorepo-jdriven
cd monorepo-jdriven/
git init --initial-branch "main" .
# Use the merge driver "union" for conflict resolution of `.gitmodules` file.
# The "union" driver simply appends new content to the file.
# https://git-scm.com/docs/gitattributes#Documentation/gitattributes.txt-union
echo '.gitmodules merge=union' >.gitattributes
git add .
git commit --message "monorepo: Initial commit"
function merge_repo {
merge_repo_jdriven "$@"
}
}
function prepare_merge_subtree {
mkdir monorepo-subtree
cd monorepo-subtree/
git init --initial-branch "main" .
git commit --allow-empty --message "monorepo: Initial commit"
function merge_repo {
merge_repo_subtree "$@"
}
}
# Choose only one...
#prepare_merge_gstreamer
#prepare_merge_jdriven
prepare_merge_subtree
merge_repo "kms-omni-build" "https://github.com/Kurento/kms-omni-build.git" "7.0.0" "server"
merge_repo "kurento-module-creator" "https://github.com/Kurento/kurento-module-creator.git" "7.0.0" "server/module-creator"
merge_repo "kms-cmake-utils" "https://github.com/Kurento/kms-cmake-utils.git" "7.0.0" "server/cmake-utils"
merge_repo "kms-jsonrpc" "https://github.com/Kurento/kms-jsonrpc.git" "7.0.0" "server/jsonrpc"
merge_repo "kms-core" "https://github.com/Kurento/kms-core.git" "7.0.0" "server/plugin-core"
merge_repo "kms-elements" "https://github.com/Kurento/kms-elements.git" "7.0.0" "server/plugin-elements"
merge_repo "kms-filters" "https://github.com/Kurento/kms-filters.git" "7.0.0" "server/plugin-filters"
merge_repo "kurento-media-server" "https://github.com/Kurento/kurento-media-server.git" "7.0.0" "server/media-server"
merge_repo "kurento-maven-plugin" "https://github.com/Kurento/kurento-maven-plugin.git" "7.0.0" "clients/java/maven-plugin"
merge_repo "kms-chroma" "https://github.com/Kurento/kms-chroma.git" "7.0.0" "server/plugin-examples/chroma"
merge_repo "kms-crowddetector" "https://github.com/Kurento/kms-crowddetector.git" "7.0.0" "server/plugin-examples/crowddetector"
merge_repo "kms-datachannelexample" "https://github.com/Kurento/kms-datachannelexample.git" "7.0.0" "server/plugin-examples/datachannelexample"
merge_repo "kms-gstreamer-plugin-sample" "https://github.com/Kurento/kms-gstreamer-plugin-sample.git" "7.0.0" "server/plugin-examples/gstreamer-sample"
merge_repo "kms-markerdetector" "https://github.com/Kurento/kms-markerdetector.git" "7.0.0" "server/plugin-examples/markerdetector"
merge_repo "kms-opencv-plugin-sample" "https://github.com/Kurento/kms-opencv-plugin-sample.git" "7.0.0" "server/plugin-examples/opencv-sample"
merge_repo "kms-platedetector" "https://github.com/Kurento/kms-platedetector.git" "7.0.0" "server/plugin-examples/platedetector"
merge_repo "kms-pointerdetector" "https://github.com/Kurento/kms-pointerdetector.git" "7.0.0" "server/plugin-examples/pointerdetector"
merge_repo "adm-scripts" "https://github.com/Kurento/adm-scripts.git" "master" "ci-scripts"
merge_repo "doc-kurento" "https://github.com/Kurento/doc-kurento.git" "7.0.0" "doc-kurento"
merge_repo "kurento-client-js" "https://github.com/Kurento/kurento-client-js.git" "master" "clients/javascript/client"
merge_repo "kurento-docker" "https://github.com/Kurento/kurento-docker.git" "master" "docker"
merge_repo "kurento-java" "https://github.com/Kurento/kurento-java.git" "7.0.0" "clients/java"
merge_repo "kurento-jsonrpc-js" "https://github.com/Kurento/kurento-jsonrpc-js.git" "master" "clients/javascript/jsonrpc"
merge_repo "kurento-qa-pom" "https://github.com/Kurento/kurento-qa-pom.git" "master" "clients/java/qa-pom"
merge_repo "kurento-tutorial-java" "https://github.com/Kurento/kurento-tutorial-java.git" "7.0.0" "tutorials/java"
merge_repo "kurento-tutorial-js" "https://github.com/Kurento/kurento-tutorial-js.git" "7.0.0" "tutorials/javascript-browser"
merge_repo "kurento-tutorial-node" "https://github.com/Kurento/kurento-tutorial-node.git" "7.0.0" "tutorials/javascript-node"
merge_repo "kurento-tutorial-test" "https://github.com/Kurento/kurento-tutorial-test.git" "master" "test/tutorial"
merge_repo "kurento-utils-js" "https://github.com/Kurento/kurento-utils-js.git" "master" "browser/kurento-utils-js"
echo "All done! Enjoy your monorepo ;-)"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment