Split a repository into batches to avoid `pack exceeds maximum allowed size` on push

git-split-and-push.sh

# Split a repository into batches to avoid `pack exceeds maximum allowed size` on git push

REMOTE=origin
BRANCH=$(git rev-parse --abbrev-ref HEAD)
BATCH_SIZE=500

# check if the branch exists on the remote
if git show-ref --quiet --verify refs/remotes/$REMOTE/$BRANCH; then
    # if so, only push the commits that are not on the remote already
    range=$REMOTE/$BRANCH..HEAD
else
    # else push all the commits
    range=HEAD
fi
# count the number of commits to push
n=$(git log --first-parent --format=format:x $range | wc -l)

# push each batch
for i in $(seq $n -$BATCH_SIZE 1); do
    # get the hash of the commit to push
    h=$(git log --first-parent --reverse --format=format:%H --skip $i -n1)
    echo "Pushing $h..."
    git push $REMOTE $h:refs/heads/$BRANCH
done

# push the final partial batch
git push $REMOTE HEAD:refs/heads/$BRANCH

I've made an addition to this script that better suit my needs (in my case, I had a newly-created repo with one empty commit (to allow this script to work). I wanted to add all files from a working directory to the repo in max 2gb commit sizes. You will need to manually add files larger than 2gb to the .gitignore, and use git lfs to track all files larger than 100mb. Otherwise, this script will work:

#!/bin/bash
# Usage: git-add-commit.sh <message>
# ensure any individual file is under 2gb and any file over 300mb is tracked using git-lfs before committing
# Check if a commit message was provided
if [ $# -eq 0 ]; then
  echo "Error: No commit message provided"
  exit 1
fi

# Store the commit message in a variable
message="$1"

# Function to determine the size of a file in KB
function size_in_kb() {
  local file="$1"
  if [ -f "$file" ]; then
    echo $(( $(wc -c < "$file") / 1024 ))))
  else
    echo 0
  fi
}

# Store the sum of sizes of staged files in a variable
git add -N .\/*
staged_files=$(git diff --name-only)

staged_size=0
if [ -n "$staged_files" ]; then
  staged_size=$(git diff --name-only -z | xargs -0 du -kc | tail -n 1 | awk '{print $1}')
else
  staged_size=0
fi
git reset HEAD

COUNTER=1
# Add changes in smaller chunks
while [ "$staged_size" -gt 0 ]; do
  chunk_size=0
  git add -N .\/*
  staged_files=$(git diff --name-only)
  git reset HEAD
  while IFS= read -r file; do
    file_size=$(size_in_kb "$file")
    if [ "$((chunk_size + file_size))" -le 2000000 ]; then
      chunk_size=$((chunk_size + file_size))
      git add "$file"
      #echo "$chunk_size"
    else
      break
    fi
  done <<< "$staged_files"

  echo "About to commit $COUNTER"
  git commit -m "$message (chunk $COUNTER)"
  #echo "$message (chunk $COUNTER)"
  let COUNTER++
  git add -N .\/*
  staged_files=$(git diff --name-only)

  if [ -n "$staged_files" ]; then
    staged_size=$(git diff --name-only -z | xargs -0 du -kc | tail -n 1 | awk '{print $1}')
  else
    echo "No staged files left."
    staged_size=0
  fi
  git reset HEAD
done