Search 6 POC merge script

search6-merge

#!/bin/bash

set -o errexit
set -o pipefail

function log() {
        echo 1>&2 "${@}"
}

function abort() {
        log "${@}"
        log 'Aborting.'
        exit 1
}

function success() {
        log "${@}"
        exit 0
}

function usage() {
	log -e "$(cat <<EOF
Usage:
	$0 [options] [repo_directory]

	repo_directory
		The directory to work in.
		Do not use your main repository or you may lose data.
		Defaults to creating a temporary directory.
	-h
		Print this message and exit.
	-d
		Debug mode; print some of the executed commands before executing them.
	-5 url
		Set the Search 5 remote URL.
	-6 url
		Set the Search 6 remote URL.
	-c string
		Set a string to prepend to commits instead of the default.
	-a commit_ref
		Append the commits referenced by the given ref to the resulting branch.
		
		The following remotes are available when parsing the commit refs:
		- search5: the remote passed to argument "-5"
		- search6-poc: the remote passed to argument "-6"
	-t url
		Set the target remote URL.

		If set, the script will attempt to push the merge result to a branch of that remote.
		The name of the branch is set by the -b option.
	-b branch_name
		The name of the branch to work in.

		The content of this branch will be completely disregarded and will be erased.
		If the -t option is set, this is also the name of the branch the result will be pushed to.
		Defaults to "HSEARCH-3179".
	-r
		Rewrite history of the Search 6 POC repository whenever it is beneficial (extensive renamings, rebases).

		Both methods should produce the exact same tree.
		
		Also, both methods will involve file moves (be it only because we will have to merge back Search 5 files into Search 6).
		This means the full history of a particular file will only be visible when using "git log --follow -- <file>",
		or "git log -- <file>' with the right configuration option: "git config --global log.follow 1".
		IDEs (at least IDEA) enable the "--follow" option by default when looking at the history of a file.
		
		However, the methods differ subtly in how history will look like after the merge.
		
		Pros of the "rewritten history" method:
		- Preserves history for files copied from Search 5 to the POC: with "--follow", most files in Search 6
		that were copied from Search 5 will have Search 5 commits in their history ("git log").
		- Does not involve any file renaming commit for files created in the POC:
		their full history will be visible even without "--follow".
		
		Pros of the "appended commit" method:
		- It's much faster, so it's ideal when testing.
		- It doesn't change the commit IDs.
EOF
	)"
}

function task_start() {
	log "========== ${@}"
}

function task_success() {
	log "========== OK"
}

function commit() {
	task_start "Committing..."
	git commit -m "${COMMIT_PREFIX}${*}"
	task_success
}

function init_repo() {
	task_start "Initializing git repository..."
	# Remove any leftovers from a previous attempt
	git reset . || true
	git checkout . || true
	git clean -xfd . || true
	git cherry-pick --abort || true
	git rebase --abort || true
	# Initialize
	git init
	task_success
}

function init_remote() {
	local REMOTE
	REMOTE="$1"
	local URL
	URL="$2"
	task_start "Initializing remote '$REMOTE' to '$URL'..."
	git remote remove "$REMOTE" 1>&2 2>/dev/null|| true
	git remote add "$REMOTE" "$URL"
	git remote set-url --push "$REMOTE" "Push to this remote is forbidden. Create a distinct remote."
	git fetch --no-tags "$REMOTE"
	task_success
}

function init_branch() {
	local BRANCH
	BRANCH="$1"
	local COMMIT_REF
	COMMIT_REF="$2"
	task_start "Initializing branch '$BRANCH' to '$COMMIT_REF'..."
	git clean -xfd . || true
	git checkout -B "$BRANCH" "$COMMIT_REF"
	task_success
}

# WARNING: this only works when adding exactly one level of directory nesting to the modules,
# for example mapper-pojo => mapper/pojo, but not mapper-pojo => mapper/po/jo
function create_module_dir_structure_script() {
	local ROOT
	ROOT="$1"
	shift
	local -a DIRS_BEFORE
	local -a DIRS_AFTER
	local COUNT=0

	while (( $# > 0 ))
	do
		i=$COUNT
		DIRS_BEFORE[$i]="$1"
		DIRS_AFTER[$i]="$2"
		shift 2
		(( ++COUNT ))
	done

	local MAX
	MAX=$(( COUNT - 1 ))

	# Rename references to the module directories
	echo "if [ -e '$ROOT/pom.xml' ]"
	echo "then"
	echo -n "$CMD_PERL -p -i"
	for i in $(seq 0 $MAX)
	do
		echo -n " -e 's(\Q<module>${DIRS_BEFORE[$i]}</module>\E)(<module>${DIRS_AFTER[$i]}</module>);'"
	done
	echo " '$ROOT/pom.xml'"
	echo "fi"

	for i in $(seq 0 $MAX)
	do
		# Only do the following if the module exists (it might not exist depending on the commit we're rewriting)
		echo "if [ -e '$ROOT/${DIRS_BEFORE[$i]}/pom.xml' ]"
		echo "then"
		# Add a <relativePath> element to the moved module's POM
		echo "    $CMD_PERL -p -i -e 's(^(    </parent>.*))(        <relativePath>../..</relativePath>\n\\1)' '$ROOT/${DIRS_BEFORE[$i]}/pom.xml'"
		# Move module directory
		echo "    mkdir -p '$ROOT/${DIRS_AFTER[$i]}'"
		echo "    mv -T '$ROOT/${DIRS_BEFORE[$i]}' '$ROOT/${DIRS_AFTER[$i]}'"
		echo "fi"
	done
}

function create_rename_package_script() {
	local -a PACKAGES_BEFORE
	local -a PACKAGES_AFTER
	local -a DIRS_BEFORE
	local -a DIRS_AFTER
	local COUNT=0

	while (( $# > 0 ))
	do
		i=$COUNT
		PACKAGES_BEFORE[$i]="$1"
		PACKAGES_AFTER[$i]="$2"
		shift 2
		DIRS_BEFORE[$i]="${PACKAGES_BEFORE[$i]//.//}"
		DIRS_AFTER[$i]="${PACKAGES_AFTER[$i]//.//}"
		(( ++COUNT ))
	done

	local MAX
	MAX=$(( COUNT - 1 ))

	# Rename references to the package and corresponding directory
	echo "find . \\"
	echo "'(' -name pom.xml -o -name '*.java' -o -name '*.properties' -o -path '*/resources/META-INF/services/*' ')' \\"
	echo -n "-execdir $CMD_PERL -p -i"
	for i in $(seq 0 $MAX)
	do
		echo -n " -e 's(\Q${PACKAGES_BEFORE[$i]}\E)(${PACKAGES_AFTER[$i]});' "
	done
	echo " '{}' '+' \\"
	echo ", '(' -name pom.xml -o -name '.travis.yml' -o -name 'Jenkinsfile' ')' \\"
	echo -n "-execdir $CMD_PERL -p -i"
	for i in $(seq 0 $MAX)
	do
		echo -n " -e 's(\Q${DIRS_BEFORE[$i]}\E)(${DIRS_AFTER[$i]});'"
	done
	echo " '{}' '+'"

	# Move directory content
	for i in $(seq 0 $MAX)
	do
		echo "find . \\"
		echo "'(' -path "*/${DIRS_BEFORE[$i]}" -type d ')' \\"
		echo "-exec bash -c '
				SOURCE=\"\$(readlink -f {})\"
				TARGET=\"\$(echo {} | $CMD_PERL -p -e \"s(\Q${DIRS_BEFORE[$i]}\E)(${DIRS_AFTER[$i]})\")\"
				TARGET=\"\$(readlink -m \"\$TARGET\")\"
				# Handle the case where the target is inside the source: move the target to a different name, and later restore its initial name
				TARGET_BASENAME=\"\$(basename \"\$TARGET\")\"
				IMPOSSIBLE_NAME=thisisanimpossiblename4242
				[ \"\$SOURCE/\$TARGET_BASENAME\" = \"\$TARGET\" ] && [ -e \"\$TARGET\" ] && mv -T \"\$TARGET\" \"\$SOURCE/\$IMPOSSIBLE_NAME\"
			       	mkdir -p \"\$TARGET\"
				mv \$(
					# Filter out the target, to avoid errors such as \"cannot move X to X\"
					find \"\$SOURCE\" -mindepth 1 -maxdepth 1 -not -path \"\$TARGET\"
					) \\
					\"\$TARGET\"
				[ -e \"\$TARGET/\$IMPOSSIBLE_NAME\" ] && mv -T \"\$TARGET/\$IMPOSSIBLE_NAME\" \"\$TARGET/\$TARGET_BASENAME\"
			' ';'"
		# Also move service files (META-INF/services/org.hibernate....)
		echo "find . -name '${PACKAGES_BEFORE[$i]}*' -type f \\"
		echo "-execdir bash -c '
				SOURCE=\"\$(readlink -f {})\"
				TARGET=\"\$(echo {} | $CMD_PERL -p -e \"s(\Q${PACKAGES_BEFORE[$i]}\E)(${PACKAGES_AFTER[$i]})\")\"
				TARGET=\"\$(readlink -m \"\$TARGET\")\"
				mv \"\$SOURCE\" \"\$TARGET\"
			' ';'"
	done
}

function refactor_modules_and_packages() {
	local PACKAGE_RENAMINGS=(
		org.hibernate.search.v6poc org.hibernate.search
		org.hibernate.search.engine org.hibernate.search.engine.common
		org.hibernate.search.backend org.hibernate.search.engine.backend
		org.hibernate.search.engine.backend.elasticsearch org.hibernate.search.backend.elasticsearch
		org.hibernate.search.engine.backend.lucene org.hibernate.search.backend.lucene
		org.hibernate.search.entity org.hibernate.search.engine.mapper
		org.hibernate.search.engine.mapper.pojo org.hibernate.search.mapper.pojo
		org.hibernate.search.engine.mapper.javabean org.hibernate.search.mapper.javabean
		org.hibernate.search.engine.mapper.orm org.hibernate.search.mapper.orm
		org.hibernate.search.annotations org.hibernate.search.engine.annotations
		org.hibernate.search.bridge org.hibernate.search.engine.bridge
		org.hibernate.search.cfg org.hibernate.search.engine.cfg
		org.hibernate.search.filter org.hibernate.search.engine.filter
		org.hibernate.search.indexes org.hibernate.search.engine.indexes
		org.hibernate.search.impl org.hibernate.search.engine.impl
		org.hibernate.search.jmx org.hibernate.search.engine.jmx
		org.hibernate.search.logging org.hibernate.search.engine.logging
		org.hibernate.search.query org.hibernate.search.engine.query
		org.hibernate.search.sandbox org.hibernate.search.engine.sandbox
		org.hibernate.search.search org.hibernate.search.engine.search
		org.hibernate.search.spi org.hibernate.search.engine.spi
		org.hibernate.search.spatial org.hibernate.search.engine.spatial
		org.hibernate.search.stat org.hibernate.search.engine.stat
		org.hibernate.search.store org.hibernate.search.engine.store
		org.hibernate.search.integrationtest.orm org.hibernate.search.integrationtest.mapper.orm
	)

	task_start "Refactoring modules and packages (packages renamings: ${PACKAGE_RENAMINGS[*]})..."
	local MODULE_SCRIPT
	local SCRIPT

	MODULE_REFACTORING_SCRIPT="$(create_module_dir_structure_script . \
		mapper-pojo mapper/pojo \
		orm mapper/orm \
		mapper-javabean mapper/javabean \
		mapper-protobuf mapper/protobuf \
		backend-lucene backend/lucene \
		backend-elasticsearch backend/elasticsearch
	)"
	MODULE_REFACTORING_SCRIPT+=";$(create_module_dir_structure_script integrationtest \
		mapper-pojo mapper/pojo \
		orm mapper/orm \
		mapper-javabean mapper/javabean \
		backend-tck backend/tck \
		backend-lucene backend/lucene \
		backend-elasticsearch backend/elasticsearch
	)"

	if (( REWRITE_HISTORY ))
	then
		SCRIPT="$MODULE_REFACTORING_SCRIPT"
		SCRIPT+=";$(create_rename_package_script "${PACKAGE_RENAMINGS[@]}")"
		git filter-branch \
			-f \
			--tree-filter "$SCRIPT"
	else
		bash -c "$MODULE_REFACTORING_SCRIPT"
		git add -A
		git commit -m "HSEARCH-3285 Move to a less \"flat\" directory structure"

		set "${PACKAGE_RENAMINGS[@]}"
		while (( $# > 0 ))
		do
			SCRIPT="$(create_rename_package_script "$1" "$2")"
			bash -c "$SCRIPT"
			git add -A
			git commit -m "${COMMIT_PREFIX}Rename package $1 to $2" || log "Nothing to commit when renaming package $1 to $2; skipping."
			shift 2
		done
	fi
	task_success
}

function set_version() {
	local VERSION
	VERSION="$1"
	task_start "Setting the version of Maven artifacts to '$VERSION'..."
	mvn -q versions:set -DnewVersion="$VERSION"
	git clean -f . # Remove .versionBackup files
	git add -A
	task_success
}

function set_name_and_description() {
	local POM
	POM="$1"
	local NAME
	NAME="$2"
	local DESCRIPTION
	DESCRIPTION="$3"
	task_start "Setting name and description of module '$POM'..."
	sed -Ei "$POM" -f - <<EOF
s,^    <name>.*</name>$,    <name>$NAME</name>,
s,^    <description>.*</description>$,    <description>$DESCRIPTION</description>,
EOF
	git add "$POM"
	task_success
}

function pull_conflicting_files() {
	local BRANCH
	BRANCH="$1"
	shift
	task_start "Synchronizing conflicting files '$@' with branch '$BRANCH'..."
	git checkout "$BRANCH" -- "${@}"
	git add "${@}"
	task_success
}

function remove_conflicting_files() {
	task_start "Removing conflicting files '$@'..."
	git rm "${@}"
	task_success
}

function rewrite_conflicting_files_history() {
	local REVISION
	REVISION="$1"
	shift
	task_start "Rewriting history of conflicting files '$@' to set them to their content at revision '$REVISION'..."
	git filter-branch \
		-f \
		--tree-filter "git checkout $REVISION -- ${*}"
	task_success
}

function prefix_commit_messages_without_ticket_key() {
	local TICKET_KEY
	TICKET_KEY="$1"
	shift
	local LAST_REWRITTEN_COMMIT
	LAST_REWRITTEN_COMMIT="$1"
	shift
	task_start "Rewriting history to prefix '$TICKET_KEY ' to commits without an assigned ticket key, from the initial commit until commit '$LAST_REWRITTEN_COMMIT'..."
	local TEMP_BRANCH_NAME
	TEMP_BRANCH_NAME=merge-script-rewrite-commit-messages-temp
	local CURRENT_BRANCH_NAME
	CURRENT_BRANCH_NAME="$(git rev-parse --abbrev-ref HEAD)"
	git checkout -B "$TEMP_BRANCH_NAME" "$LAST_REWRITTEN_COMMIT"
	git filter-branch \
		-f \
		--msg-filter 'perl -pi -e '"'"'if ($. == 1) { s/^(?!HSEARCH-\S+)/'"$TICKET_KEY"' / }'"'"
	git checkout "$CURRENT_BRANCH_NAME"
	git rebase --onto "$TEMP_BRANCH_NAME" "$LAST_REWRITTEN_COMMIT" "$CURRENT_BRANCH_NAME"
	task_success
}

function move_code_to_legacy_subdirectory() {
	task_start "Moving code to a new subdirectory 'legacy'..."
	if [ -e legacy ]
	then
		# Special case: the legacy directory already exists
		mkdir legacy/legacy
		find legacy -mindepth 1 -maxdepth 1 -not -path 'legacy/legacy' -print0 | $CMD_XARGS -0 -r -I'{}' mv '{}' 'legacy/legacy'
	else
		mkdir legacy
	fi
	find . -mindepth 1 -maxdepth 1 \( -type d -not -path './legacy' -not -path './.git' -o -path ./.travis.yml -o -path ./pom.xml \) -print0 | $CMD_XARGS -0 -r -I'{}' mv '{}' 'legacy'
	git add -A
	task_success
}

function set_as_submodule() {
	local PARENT_POM
	PARENT_POM="$1"; shift
	local ADD_BEFORE_MODULE
	ADD_BEFORE_MODULE="$1"; shift
	local RELATIVE_SUBMODULE_PATH
	RELATIVE_SUBMODULE_PATH="$1"; shift
	local GROUP_ID
	GROUP_ID="$1"; shift
	local ARTIFACT_ID
	ARTIFACT_ID="$1"; shift
	local VERSION
	VERSION="$1"; shift
	local SUBMODULE_POM
	SUBMODULE_POM="$(dirname $PARENT_POM)/$RELATIVE_SUBMODULE_PATH/pom.xml"
	task_start "Setting '$SUBMODULE_POM' as a submodule of '$ARTIFACT_ID'..."
	sed -Ei "$SUBMODULE_POM" -f - <<EOF
s,^(    <groupId>.*</groupId>),    <parent>\n\
        <groupId>$GROUP_ID</groupId>\n\
        <artifactId>$ARTIFACT_ID</artifactId>\n\
        <version>$VERSION</version>\n\
    </parent>\n\
\1,
/^    <version>.*<\/version>$/d
EOF
	sed -Ei "s,((.*)<module>$ADD_BEFORE_MODULE</module>.*),\2<module>$RELATIVE_SUBMODULE_PATH</module>\n\1," "$PARENT_POM"
	git add "$SUBMODULE_POM" "$PARENT_POM"
	task_success
}

function set_artifact_id() {
	local POM
	POM="$1"
	local OLD_ARTIFACT_ID
	OLD_ARTIFACT_ID="$2"
	local NEW_ARTIFACT_ID
	NEW_ARTIFACT_ID="$3"
	local POM_DIRECTORY
	POM_DIRECTORY="$(dirname $POM)"
	task_start "Setting artifact ID of '$POM' to '$NEW_ARTIFACT_ID'..."
	sed -Ei "s/^(    <artifactId>)$OLD_ARTIFACT_ID/\1$NEW_ARTIFACT_ID/" "$POM"
	find "$POM_DIRECTORY" -mindepth 1 -name 'pom.xml' -print0 | $CMD_XARGS -0 -r -I'{}' sed -Ei "s/(<artifactId>)$OLD_ARTIFACT_ID/\1$NEW_ARTIFACT_ID/" '{}'
	git add -A
	task_success
}

CMD_PERL=$(which perl)
CMD_XARGS=$(which xargs)

SEARCH6_POC_URL=https://github.com/hibernate/hibernate-search-6-poc.git
SEARCH5_URL=https://github.com/hibernate/hibernate-search.git
COMMIT_PREFIX="HSEARCH-3179 "
REWRITE_HISTORY=0
TARGET_BRANCH="HSEARCH-3179"

while getopts '5:6:c:rhda:t:b:' opt
do
        case "$opt" in
                5)
                        SEARCH5_URL="$OPTARG"
                        ;;
                6)
                        SEARCH6_POC_URL="$OPTARG"
                        ;;
                c)
                        COMMIT_PREFIX="$OPTARG"
                        ;;
		r)
			REWRITE_HISTORY=1
			;;
		d)
			CMD_XARGS="$CMD_XARGS -t"
			;;
		h)
			usage
			exit 0
			;;
                a)
                        APPEND_REFS="$OPTARG"
                        ;;
                t)
                        TARGET_URL="$OPTARG"
                        ;;
                b)
                        TARGET_BRANCH="$OPTARG"
                        ;;
                \?)
                        exit 1
                        ;;
        esac
done

shift $(( OPTIND - 1 ))

if [ $# = 1 ]
then
	WORKING_DIR=$1
	shift
elif [ $# = 0 ]
then
	WORKING_DIR=$(mktemp --tmpdir -d search6-merge-XXX)
	trap "(( ! SUCCESS )) && echo 1>&2 'Error detected, removing $WORKING_DIR' && rm -rf '$WORKING_DIR'" EXIT
else
	usage
	abort
fi

log "Working in directory $WORKING_DIR"
cd "$WORKING_DIR"

init_repo
init_remote search5 "$SEARCH5_URL"
init_remote search6-poc "$SEARCH6_POC_URL"

init_branch search5-to-merge search5/master
set_version 6.0.0-SNAPSHOT # Current version is 5.<something>
commit "Set version to '6.0.0-SNAPSHOT'"
move_code_to_legacy_subdirectory
commit "Move code to a legacy directory to avoid conflicts when merging"

init_branch search6-poc-to-merge search6-poc/master
refactor_modules_and_packages
set_name_and_description ./pom.xml "Hibernate Search Parent POM" "Hibernate Search Parent POM"
commit "Remove all mentions of this project being a POC"
set_version 6.0.0-SNAPSHOT # Current version is 6.0-SNAPSHOT (only two components)
commit "Set version to '6.0.0-SNAPSHOT'"

if (( REWRITE_HISTORY ))
then
	# We are allowed to rewrite history, so let's rebase first:
	# this will allow to preserve history for files copied from Search 5 into the POC, in particular

	# First make sure we won't have any conflict
	# We'd rather keep these files in the repository at all time, and we want to keep the Search 5 version,
	# so we'll set them to the Search 5 version in the Search 6 POC history
	rewrite_conflicting_files_history search5-to-merge copyright.txt lgpl.txt README.md .gitignore changelog.txt
	# Then take advantage of this opportunity to rewrite commit messages up until a certain commit
	prefix_commit_messages_without_ticket_key HSEARCH-3302 "HEAD^{/HSEARCH-3160 Move Sonar token definition to the Travis repository and skip Sonar execution if no token is defined}"
	# Then do the actual rebase
	git rebase --onto search5-to-merge --root search6-poc-to-merge

	init_branch "$TARGET_BRANCH" search5-to-merge
	git merge -q --ff-only search6-poc-to-merge
else
	# We are not allowed to rewrite history, so rebase is forbidden
	# Let's just synchronize a few files that would otherwise trigger a merging conflict
	git checkout search6-poc-to-merge
	pull_conflicting_files search5-to-merge copyright.txt lgpl.txt README.md .gitignore changelog.txt
	commit "Synchronize conflicting files with the main repository before the merge"

	init_branch "$TARGET_BRANCH" search5-to-merge
	git merge -q --allow-unrelated-histories --no-ff search6-poc-to-merge \
		-m "${COMMIT_PREFIX}Merge the Hibernate Search 6 proof-of-concept into the main repository"
fi

set_name_and_description './legacy/pom.xml' "Hibernate Search Legacy Modules Parent POM" "Hibernate Search Legacy Modules Parent POM"
set_artifact_id './legacy/pom.xml' 'hibernate-search-parent' 'hibernate-search-legacy-parent'
set_as_submodule './pom.xml' 'reports' 'legacy' 'org.hibernate.search' 'hibernate-search-parent' '6.0.0-SNAPSHOT'
commit "Set directory 'legacy' as a legacy submodule of the main project"

if [ -n "$APPEND_REFS" ]
then
	git cherry-pick "$APPEND_REFS"
fi

if [ -n "$TARGET_URL" ]
then
	git push --force "$TARGET_URL" "$TARGET_BRANCH"
fi

log "Merge is successful, see the result in $WORKING_DIR"
log "Some manual adjustments will be necessary:
	- Merge most configuration (properties, plugins, ...) from ./legacy/pom.xml into ./pom.xml
	- Merge Travis configuration (legacy/.travis.yml and legacy/travis)
	- Merge Jenkins configuration (legacy/jenkins)
	- Merge modules that only make sense at the root (build-config, distribution, reports)
	- Merge 'legacy/sharedtestresources' into 'util/internal/integrationtest/sharedresources'
	- Later, merge code from legacy modules into the new ones"

SUCCESS=1
exit 0