cedricwalter/mergeGitRepositories.sh

## mergeGitRepositories.sh
#!/bin/bash
#
################################################################################
## Script to merge multiple git repositories into a new repository
## - The new repository will contain a folder for every merged repository
## - The script adds remotes for every project and then merges in every branch
##   and tag. These are renamed to have the origin project name as a prefix
##
## Usage: mergeGitRepositories.sh <new_project> <my_repo_urls.lst>
## - where <new_project> is the name of the new project to create
## - and <my_repo_urls.lst> is a file contaning the URLs to the respositories
##   which are to be merged on separate lines.
##
## Version: 0.2.0 Created: 2015-06-17 Robert von Burg eitch@eitchnet.ch
## Version: 0.2.1 Created: 2016-07-14 Cédric Walter www.cedricwalter.com
##
################################################################################
#

# disallow using undefined variables
shopt -s -o nounset

# Script variables
declare SCRIPT_NAME="${0##*/}"
declare SCRIPT_DIR="$(cd ${0%/*} ; pwd)"
declare ROOT_DIR="$PWD"


# Detect proper usage
if [ "$#" -ne "2" ] ; then
  echo -e "ERROR: Usage: $0 <new_project> <my_repo_urls.lst>"
  exit 1
fi


# Script functions
function failed() {
  echo -e "ERROR: Merging of projects failed:"
  echo -e "$1"
  exit 1
}

function commit_merge() {
  current_branch="$(git symbolic-ref HEAD 2>/dev/null)"
  CHANGES=$(git status | grep "working directory clean")
  MERGING=$(git status | grep "merging")
  if [[ "$CHANGES" != "" ]] && [[ "$MERGING" == "" ]] ; then
    echo -e "INFO:   No commit required."
  else
    echo -e "INFO:   Committing ${sub_project}..."
    if ! git commit --quiet -m "[Project] Merged branch '$1' of ${sub_project}" ; then
      failed "Failed to commit merge of branch '$1' of ${sub_project} into ${current_branch}"
    fi
  fi
}


## Script variables
PROJECT_NAME="${1}"
PROJECT_PATH="${ROOT_DIR}/${PROJECT_NAME}"
REPO_FILE="${2}"
REPO_URL_FILE="${ROOT_DIR}/${REPO_FILE}"


# Make sure the REPO_URL_FILE exists
if [ ! -e "${REPO_URL_FILE}" ] ; then
  echo -e "ERROR: Repo file ${REPO_URL_FILE} does not exist!"
  exit 1
fi


# Make sure the required directories don't exist
if [ -e "${PROJECT_PATH}" ] ; then
  echo -e "ERROR: Project ${PROJECT_NAME} already exists!"
  exit 1
fi


# create the new project
echo -e "INFO: Creating new git repository ${PROJECT_NAME}..."
echo -e "===================================================="
cd ${ROOT_DIR}
mkdir ${PROJECT_NAME}
cd ${PROJECT_NAME}
git init
echo "Initial Commit" > initial_commit
# Since this is a new repository we need to have at least one commit
# thus were we create temporary file, but we delete it again.
# Deleting it guarantees we don't have conflicts later when merging
git add initial_commit
git commit --quiet -m "[Project] Initial Master Repo Commit"
git rm --quiet initial_commit
git commit --quiet -m "[Project] Initial Master Repo Commit"
echo


# Merge all projects into th branches of this project
echo -e "INFO: Merging projects into new repository..."
echo -e "===================================================="
for url in $(cat ${REPO_URL_FILE}) ; do

  # extract the name of this project
  export sub_project=${url##*/}
  export sub_project=${sub_project%.*}

  echo -e "Work on Project: ${sub_project}"
  echo -e "Work on URL: ${url}"
  echo -e "----------------------------------------------------"

  # Fetch the project
  echo "INFO:   Fetching ${sub_project}"
  echo "git remote add ${sub_project} ${url}"
  git remote add ${sub_project} ${url}
  echo git fetch --tags ${sub_project}
  if ! git fetch --tags ${sub_project} 2>/dev/null ; then
    failed "Failed to fetch project ${sub_project}"
  fi
  if ! git fetch --all 2>/dev/null ; then
    failed "Failed to fetch project ${sub_project}"
  fi

  # add remote branches
  echo -e "INFO:   Creating local branches for ${sub_project}..."
  while read branch ; do
    branch_ref=$(echo $branch | tr " " "\t" | cut -f 1)
    branch_name=$(echo $branch | tr " " "\t" | cut -f 2 | cut -d / -f 3-)

    echo -e "INFO:   Creating branch ${branch_name}..."

    # create and checkout new merge branch off of master
    git checkout --quiet -b "${sub_project}/${branch_name}" master
    git reset --hard --quiet
    git clean -d --force --quiet

    # Merge the project
    echo -e "INFO:   Merging ${sub_project}..."
    if ! git merge --allow-unrelated-histories --quiet --no-commit "remotes/${sub_project}/${branch_name}" 2>/dev/null ; then
      failed "Failed to merge branch 'remotes/${sub_project}/${branch_name}' from ${sub_project}"
    fi

    # And now see if we need to commit (maybe there was a merge)
    commit_merge "${sub_project}/${branch_name}"

    # relocate projects files into own directory
    if [ "$(ls)" == "${sub_project}" ] ; then
      echo -e "WARN:   Not moving files in branch ${branch_name} of ${sub_project} as already only one root level."
    else
      echo -e "INFO:   Moving files in branch ${branch_name} of ${sub_project} so we have a single directory..."
      mkdir ${sub_project}
      for dir in ./* ; do
	  dir=${dir%*/}
	  echo "list: ${dir}"
        if  [[ "$dir" == "${sub_project}" ]] ||
            [[ "$dir" == "." ]] ||
            [[ "$dir" == ".." ]] ; then
          continue
        fi
        echo -E "git mv -k "$dir" "${sub_project}/""
		git mv -k "$dir" "${sub_project}/"
      done
	  git mv -k .git "${sub_project}/"

      # commit the moving
      if ! git commit --quiet -m  "[Project] Move ${sub_project} files into sub directory" ; then
        failed "Failed to commit moving of ${sub_project} files into sub directory"
      fi
    fi
    echo
  done < <(git ls-remote --heads ${sub_project})


  # checkout master of sub probject
  if ! git checkout "${sub_project}/master" 2>/dev/null ; then
    failed "sub_project ${sub_project} is missing master branch!"
  fi

  # copy remote tags
  echo -e "INFO:   Copying tags for ${sub_project}..."
  while read tag ; do
    tag_ref=$(echo $tag | tr " " "\t" | cut -f 1)
    tag_name=$(echo $tag | tr " " "\t" | cut -f 2 | cut -d / -f 3)

    # hack for broken tag names where they are like 1.2.0^{} instead of just 1.2.0
    tag_name="${tag_name%%^*}"

    tag_new_name="${sub_project}/${tag_name}"
    echo -e "INFO:     Copying tag ${tag_name} to ${tag_new_name} for ref ${tag_ref}..."
    if ! git tag "${tag_new_name}" "${tag_ref}" 2>/dev/null ; then
      echo -e "WARN:     Could not copy tag ${tag_name} to ${tag_new_name} for ref ${tag_ref}"
    fi
  done < <(git ls-remote --tags ${sub_project})

  # Remove the remote to the old project
  echo -e "INFO:   Removing remote ${sub_project}..."
  git remote rm ${sub_project}

  echo
done


# Now merge all project master branches into new master
git checkout --quiet master
echo -e "INFO: Merging projects master branches into new repository..."
echo -e "===================================================="
for url in $(cat ${REPO_URL_FILE}) ; do

  # extract the name of this project
  export sub_project=${url##*/}
  sub_project=${sub_project%*.git}

  echo -e "INFO:   Merging ${sub_project}..."
  if ! git merge --quiet --no-commit "${sub_project}/master" 2>/dev/null ; then
    failed "Failed to merge branch ${sub_project}/master into master"
  fi

  # And now see if we need to commit (maybe there was a merge)
  commit_merge "${sub_project}/master"

  echo
done


# Done
cd ${ROOT_DIR}
echo -e "INFO: Done."
echo

exit 0
	#!/bin/bash
	#
	################################################################################
	## Script to merge multiple git repositories into a new repository
	## - The new repository will contain a folder for every merged repository
	## - The script adds remotes for every project and then merges in every branch
	## and tag. These are renamed to have the origin project name as a prefix
	##
	## Usage: mergeGitRepositories.sh <new_project> <my_repo_urls.lst>
	## - where <new_project> is the name of the new project to create
	## - and <my_repo_urls.lst> is a file contaning the URLs to the respositories
	## which are to be merged on separate lines.
	##
	## Version: 0.2.0 Created: 2015-06-17 Robert von Burg eitch@eitchnet.ch
	## Version: 0.2.1 Created: 2016-07-14 Cédric Walter www.cedricwalter.com
	##
	################################################################################
	#

	# disallow using undefined variables
	shopt -s -o nounset

	# Script variables
	declare SCRIPT_NAME="${0##*/}"
	declare SCRIPT_DIR="$(cd ${0%/*} ; pwd)"
	declare ROOT_DIR="$PWD"


	# Detect proper usage
	if [ "$#" -ne "2" ] ; then
	echo -e "ERROR: Usage: $0 <new_project> <my_repo_urls.lst>"
	exit 1
	fi


	# Script functions
	function failed() {
	echo -e "ERROR: Merging of projects failed:"
	echo -e "$1"
	exit 1
	}

	function commit_merge() {
	current_branch="$(git symbolic-ref HEAD 2>/dev/null)"
	CHANGES=$(git status \| grep "working directory clean")
	MERGING=$(git status \| grep "merging")
	if [[ "$CHANGES" != "" ]] && [[ "$MERGING" == "" ]] ; then
	echo -e "INFO: No commit required."
	else
	echo -e "INFO: Committing ${sub_project}..."
	if ! git commit --quiet -m "[Project] Merged branch '$1' of ${sub_project}" ; then
	failed "Failed to commit merge of branch '$1' of ${sub_project} into ${current_branch}"
	fi
	fi
	}


	## Script variables
	PROJECT_NAME="${1}"
	PROJECT_PATH="${ROOT_DIR}/${PROJECT_NAME}"
	REPO_FILE="${2}"
	REPO_URL_FILE="${ROOT_DIR}/${REPO_FILE}"


	# Make sure the REPO_URL_FILE exists
	if [ ! -e "${REPO_URL_FILE}" ] ; then
	echo -e "ERROR: Repo file ${REPO_URL_FILE} does not exist!"
	exit 1
	fi


	# Make sure the required directories don't exist
	if [ -e "${PROJECT_PATH}" ] ; then
	echo -e "ERROR: Project ${PROJECT_NAME} already exists!"
	exit 1
	fi


	# create the new project
	echo -e "INFO: Creating new git repository ${PROJECT_NAME}..."
	echo -e "===================================================="
	cd ${ROOT_DIR}
	mkdir ${PROJECT_NAME}
	cd ${PROJECT_NAME}
	git init
	echo "Initial Commit" > initial_commit
	# Since this is a new repository we need to have at least one commit
	# thus were we create temporary file, but we delete it again.
	# Deleting it guarantees we don't have conflicts later when merging
	git add initial_commit
	git commit --quiet -m "[Project] Initial Master Repo Commit"
	git rm --quiet initial_commit
	git commit --quiet -m "[Project] Initial Master Repo Commit"
	echo


	# Merge all projects into th branches of this project
	echo -e "INFO: Merging projects into new repository..."
	echo -e "===================================================="
	for url in $(cat ${REPO_URL_FILE}) ; do

	# extract the name of this project
	export sub_project=${url##*/}
	export sub_project=${sub_project%.*}

	echo -e "Work on Project: ${sub_project}"
	echo -e "Work on URL: ${url}"
	echo -e "----------------------------------------------------"

	# Fetch the project
	echo "INFO: Fetching ${sub_project}"
	echo "git remote add ${sub_project} ${url}"
	git remote add ${sub_project} ${url}
	echo git fetch --tags ${sub_project}
	if ! git fetch --tags ${sub_project} 2>/dev/null ; then
	failed "Failed to fetch project ${sub_project}"
	fi
	if ! git fetch --all 2>/dev/null ; then
	failed "Failed to fetch project ${sub_project}"
	fi

	# add remote branches
	echo -e "INFO: Creating local branches for ${sub_project}..."
	while read branch ; do
	branch_ref=$(echo $branch \| tr " " "\t" \| cut -f 1)
	branch_name=$(echo $branch \| tr " " "\t" \| cut -f 2 \| cut -d / -f 3-)

	echo -e "INFO: Creating branch ${branch_name}..."

	# create and checkout new merge branch off of master
	git checkout --quiet -b "${sub_project}/${branch_name}" master
	git reset --hard --quiet
	git clean -d --force --quiet

	# Merge the project
	echo -e "INFO: Merging ${sub_project}..."
	if ! git merge --allow-unrelated-histories --quiet --no-commit "remotes/${sub_project}/${branch_name}" 2>/dev/null ; then
	failed "Failed to merge branch 'remotes/${sub_project}/${branch_name}' from ${sub_project}"
	fi

	# And now see if we need to commit (maybe there was a merge)
	commit_merge "${sub_project}/${branch_name}"

	# relocate projects files into own directory
	if [ "$(ls)" == "${sub_project}" ] ; then
	echo -e "WARN: Not moving files in branch ${branch_name} of ${sub_project} as already only one root level."
	else
	echo -e "INFO: Moving files in branch ${branch_name} of ${sub_project} so we have a single directory..."
	mkdir ${sub_project}
	for dir in ./* ; do
	dir=${dir%*/}
	echo "list: ${dir}"
	if [[ "$dir" == "${sub_project}" ]] \|\|
	[[ "$dir" == "." ]] \|\|
	[[ "$dir" == ".." ]] ; then
	continue
	fi
	echo -E "git mv -k "$dir" "${sub_project}/""
	git mv -k "$dir" "${sub_project}/"
	done
	git mv -k .git "${sub_project}/"

	# commit the moving
	if ! git commit --quiet -m "[Project] Move ${sub_project} files into sub directory" ; then
	failed "Failed to commit moving of ${sub_project} files into sub directory"
	fi
	fi
	echo
	done < <(git ls-remote --heads ${sub_project})


	# checkout master of sub probject
	if ! git checkout "${sub_project}/master" 2>/dev/null ; then
	failed "sub_project ${sub_project} is missing master branch!"
	fi

	# copy remote tags
	echo -e "INFO: Copying tags for ${sub_project}..."
	while read tag ; do
	tag_ref=$(echo $tag \| tr " " "\t" \| cut -f 1)
	tag_name=$(echo $tag \| tr " " "\t" \| cut -f 2 \| cut -d / -f 3)

	# hack for broken tag names where they are like 1.2.0^{} instead of just 1.2.0
	tag_name="${tag_name%%^*}"

	tag_new_name="${sub_project}/${tag_name}"
	echo -e "INFO: Copying tag ${tag_name} to ${tag_new_name} for ref ${tag_ref}..."
	if ! git tag "${tag_new_name}" "${tag_ref}" 2>/dev/null ; then
	echo -e "WARN: Could not copy tag ${tag_name} to ${tag_new_name} for ref ${tag_ref}"
	fi
	done < <(git ls-remote --tags ${sub_project})

	# Remove the remote to the old project
	echo -e "INFO: Removing remote ${sub_project}..."
	git remote rm ${sub_project}

	echo
	done


	# Now merge all project master branches into new master
	git checkout --quiet master
	echo -e "INFO: Merging projects master branches into new repository..."
	echo -e "===================================================="
	for url in $(cat ${REPO_URL_FILE}) ; do

	# extract the name of this project
	export sub_project=${url##*/}
	sub_project=${sub_project%*.git}

	echo -e "INFO: Merging ${sub_project}..."
	if ! git merge --quiet --no-commit "${sub_project}/master" 2>/dev/null ; then
	failed "Failed to merge branch ${sub_project}/master into master"
	fi

	# And now see if we need to commit (maybe there was a merge)
	commit_merge "${sub_project}/master"

	echo
	done


	# Done
	cd ${ROOT_DIR}
	echo -e "INFO: Done."
	echo

	exit 0