Monorepo creator

create-monorepo.sh

#!/bin/bash

# This removes a warning from Git to avoid delay
export FILTER_BRANCH_SQUELCH_WARNING=1;

new_repo=~/dev/ado/blue-workspace
mkdir -p "$new_repo"
cd "$new_repo"
git init

# Empty commit to master
git commit -m "Monorepo Initial commit" --allow-empty

declare sources;

# Format: repo unique identifier;repo source URL; folder where code from repo will live in monorepo
# As many sources as required can be added
sources[0]="repo-identifier-one;https://github.com/manrueda/repo-one.git;libs/one";
sources[1]="repo-identifier-two;https://github.com/manrueda/repo-two.git;libs/two";
sources[2]="repo-identifier-three;https://github.com/manrueda/repo-three;libs/three";

# Collect all master branches that will be merge into the monorepo master branch
# Format: <branch-name> <branch-name> <branch-name> <etc>
MASTERS_TO_MERGE="";

for source in ${sources[*]}
do
  # Decompose the source format into the 3 parts (identifier, remore URL and destination folder)
  IFS=';' read -ra sections <<< "$source"
  identifier="${sections[0]}";
  remoteUrl="${sections[1]}";
  destinationDir="${sections[2]}";
  remoteName="$identifier-remote";

  echo "---> moving code from remote $remoteUrl (as $remoteName) into $destinationDir"

  # Adds master for this remote to the merging list
  MASTERS_TO_MERGE="$MASTERS_TO_MERGE $identifier/master";

  # Add remote and pull the code
  git remote add $remoteName $remoteUrl
  git fetch --all

  # List all branches from the remote source and pull them as local branches
  # (With the IF it can filter out branches)
  git for-each-ref --format="%(refname)" refs/remotes/$remoteName --no-merged | \
    sed "s/refs\/remotes\/$remoteName\///" | \
    while read branchName
    do
      if [ $branchName != "I-DONT-WANT-YOU" ]
      then
        git branch "$identifier/$branchName" "$remoteName/$branchName";
      fi
    done

  # Clean file that will track tags related to all this branches
  # Will be used to remove them from the repo
  echo "" > ~/.merge-repos-tag-to-delete.data;

  # filter-branch allows to perform operations over all commits and tags that are related to the branches
  #
  # --index-filter moves all files from the root to the destination directory
  #
  # --tag-name-filter rename tags, prefixing it with the identifier.
  # Also save the original tag name to ~/.merge-repos-tag-to-delete.data to delete it later
  #
  # --prune-empty removes empty commits that may result from the re-write
  #
  # -- --branches filter makes only re-write commit over the branches that are prefixed with the identifier
  # (Becasue we always prefix the branches when we clone them from the remote, this is a safe way to narrow the work)
  git filter-branch \
      -f \
      --index-filter "
        git ls-files -s | sed \"s/	/	${destinationDir//\//\\\/}\//\" |
          GIT_INDEX_FILE=\$GIT_INDEX_FILE.new \
            git update-index --index-info &&
        mv \"\$GIT_INDEX_FILE.new\" \"\$GIT_INDEX_FILE\"" \
      --tag-name-filter "
        export tagName=\$(cat) && \
        echo "\$tagName" >> ~/.merge-repos-tag-to-delete.data;
        echo \$tagName | sed s/^/$identifier\\\//" \
      --prune-empty \
      -- --branches="$identifier/*"

  # Reads each tag from ~/.merge-repos-tag-to-delete.data and delete it
  while read LINE
      do git tag -d $LINE
  done < ~/.merge-repos-tag-to-delete.data

  # Clean-up the tag-to-delete data
  rm ~/.merge-repos-tag-to-delete.data;

  # Removes the remote because we have all the data locally now (also to avoid potential pushes)
  git remote rm $remoteName
  # Remove all "original" refs that filter-branch creates.
  # This will make the repo way smaller but there is no way to undo the merge into monorepo.
  git for-each-ref --format="delete %(refname)" refs/original | git update-ref --stdin
  # Makes all refs that are not being references anymore go away
  git reflog expire --expire=now --all
  # Force garbage collection to release the bytes
  git gc --prune=now
done

# Decompose MASTERS_TO_MERGE into an array of items
IFS=';' read -ra MASTERS_TO_MERGE_ARR <<< "$MASTERS_TO_MERGE"

# Starts a octopus merge between all master branches into the monorepo master branch
# (it will "fail" but it's just because the unrelated history, the following steps make it work)
git merge --allow-unrelated-histories $MASTERS_TO_MERGE;

# Makes the result of the merge appear in the monorepo master branch
# Has to be done branch by branch because there is a limit of 8 branches at the same time.
for BRANCH in ${MASTERS_TO_MERGE_ARR[*]}
do
  echo "---> read-tree of $BRANCH"
  git read-tree $BRANCH;
  git checkout-index -a;
done

# Adds the changes and commits the merge
git add .;
git commit -m "Monorepo octopus merge between branches: $MASTERS_TO_MERGE";

# Resets to the merge result and then remove all the merged master branches only leaving the monorepo one
git reset --hard;
git branch -d $MASTERS_TO_MERGE;