Using rsync --exclude-from / --include-from / --backup-dir to facilitate safely and fully templating "projects"

template.manifest

# For very basic cases, you shouldn't need to worry about such an extensive / granular
# inclusion file, unless you're actually *using* the "template"; if you are, you should
# probably stop doing that :>
- __pycache__
- *.py[cod]
- *$py.class
- *.pyc
- .git
- README.md
# Exclude template script itself...
- template.sh

# The list of files to be copied, some
+ COPYING
+ .pre-commit-config.yaml
+ .flake8
+ pyproject.toml
+ .gitignore
+ Makefile
+ setup.py
+ setup.cfg

+ etc
+ etc/interactive
+ etc/pip*.ini
+ etc/pip.ini.*
+ etc/.gitignore

+ venv
+ venv/.gitignore
+ venv/*constraints*.txt
+ venv/*requirements*.txt*
+ venv/*constraints*.txt*
+ venv/*requirements*.txt*

+ packages/**

- **

template.sh

#!/bin/bash
#
# Use a git repository as a template to initialize a new repository
# This depends upon a file named template.manifest in the template repository
# to act as a manifest
#
# This file (template.sh) and the manifest (template.manifest) must be in the root
# of your template project. To apply the template to a new repository, clone the
# template locally, then use:
#
#   $ ./template.sh <new project git url> <local directory>
#
# (C) 2020, github@mzpqnxow.com under the terms of the 3-Clause BSD License
#
# You can do this in a much simpler fashion if you only have a few files in your
# template repository, and/or they don't change much, or only have minor changes
# over time. This method/script is especially suited for templates that have a very
# complex structure that may change over time, and has files that may need to be
# excluded. It uses `rsync --include-from=...` for the copy of the files into the
# new repository, which makes the git add/commit operations very precise
#
# Usage:
#   1. Clone the template repository, which must contain at least two files:
#     - template.sh (this script)
#     - template.manifest (in rsync --include-from format)
#       - See the example in the gist, or in my other project which makes use of --include-from
#         - https://gist.github.com/mzpqnxow/531bf97f7b8ae9d46eca30c4bd2b9017
#         - https://github.com/mzpqnxow/pyboot3/blob/master/.rsync_include.lst
#   2. Create the new (empty) repository in gogs, let's say the repository
#      is ssh://gogs.private/user/newproject
#   3. Invoke template.sh with the arguments:
#     - ./template.sh ssh://gogs.private/user/newproject ~/projects/newproject
#   4. Push the changes made by this script
#     - pushd ~/projects/newproject && git push
#
set -eu
set -o pipefail
# set -x

# Set to 0 to disable add/commit
declare -r GIT_ADD=1
declare -r GIT_COMMIT=1
# NOTE: Push is always manual
declare -r PROJECTS_DIR=~/projects/test-template

# See man rsync or the example for the format of this file
# It is used with `rsync --include-from ...`
#
# Most cases probably don't need this and could simply use
# something like `cp $template/*` or `cp $template/.*` but making
# use of rsync has some perks, including allowing git add/commit
# with precisely the files that have been moved from the template
# This file should be in the root of your "template" project, which
# should be in the same directory as this script
declare -r TEMPLATE_MANIFEST="template.manifest"

# Because the template part of your repo may not change often and/or
# may not want changes to that part to be committed back when they
# do, you can have them in .gitignore. For this, keep FORCE=-f
declare -r FORCE=-f
# declare -r FORCE=

assert_perl_grep() {
  echo OK | grep -Po '^OK' &>/dev/null || return 1
}

assert_cmd() {
  if ! command -v "$1" >/dev/null; then
    echo "FAIL: Required '$1' not present"
    return 1
  fi
}


if [ $# -lt 2 ]; then
  echo "Usage:"
  echo "  $0 <git url> <local dst directory>"
  exit 1
fi


if [ $GIT_COMMIT -eq 1 ] && [ $GIT_ADD -eq 0 ]; then
  echo "Can not commit with add disabled, makes no sense :>"
  exit 1
fi

assert_cmd rsync
assert_cmd git
if ! assert_perl_grep; then
  echo "Your system grep does not support Perl regex (grep -P), sorry"
  exit 1
fi

declare -r TEMPLATE_NAME="$(git config --get remote.origin.url)"
declare -r NEW_REPO_GIT_URL="$1"
declare -r NEW_REPO_NAME="$(basename "$NEW_REPO_GIT_URL")"


echo "Applying template to new repository $()"
echo "  Template Repository:             $TEMPLATE_NAME"
echo "  Template Repository Manifest:    $TEMPLATE_MANIFEST"
echo "  New Repository:                  $NEW_REPO_GIT_URL"
echo
read -p "Enter to continue with templating process, control-c to abort ..."

# Make temporary directories
declare -r REPOTMPDIR="$(mktemp -d --suffix=.git)"
declare -r RSLOG="$(mktemp --suffix=.log)"
trap "rm -rf $REPOTMPDIR $RSLOG" EXIT ERR

# Ensure the projects directory exists
mkdir -p "$PROJECTS_DIR"

# Final location for the new repository with the files added
# If no destination is provided as the second argument to the script,
# a temporary directory in the $PROJECTS_DIR directory will be used
# as the final resting place
declare -r REPODIR="$2"

if [ -d "$REPODIR" ]; then
  echo "Directory $REPODIR already exists!"
  echo "Exiting, will not clobber it!"
  exit 1
fi

# Clone the new repository to the temporary directory
git clone "$NEW_REPO_GIT_URL" "$REPOTMPDIR/"
# Use rsync to copy the exact files from the template repository
# Use the log file for git add/git commit later
if [ -f "$TEMPLATE_MANIFEST" ]; then
  rsync -a --log-file="$RSLOG" --log-file-format='|%n' --include-from="$TEMPLATE_MANIFEST" . "$REPOTMPDIR"
else
  echo "No template manifest file ($TEMPLATE_MANIFEST) is present!"
  exit 1
fi

if [ ! -d "$(dirname "$REPODIR")" ]; then
  read -p "WARNING: Projects directory doesn't exist; we will create it, enter to continue ..."
  mkdir -p "$(dirname "$REPODIR")"
fi

# If you want to allow this to work when the directory is already checked
# out you can use rsync instead of cp, ensure you have a trailing slash on
# both the src and dst, otherwise it will create a directory with the files
# instead of copying just the files
#
# rsync -a "$REPOTMPDIR/" "$REPODIR/"

# Otherwise just copy the tmp directory into place
cp -a "$REPOTMPDIR" "$REPODIR"
# Inspect the new repo, add/commit files from template ...
pushd "$REPODIR"
echo "Status of new project after template files copied:"
echo
git status
echo


# A precise list of files that were added from the template was emitted
# by rsync so the `git add` and `git commit` will not include/exclude
# any files
declare -r TEMPLATE_FILES="$(grep -Po '(?<=\|)(?!\.\/).*' "$RSLOG")"
echo "New Repository: $NEW_REPO_NAME"
echo "New Repository Local Location: $REPODIR"
echo
echo "--- Files Copied From Template ---"
echo
echo $TEMPLATE_FILES | tr ' ' '\n' | sed -e $'s|^|\t|'
echo



if [ $GIT_ADD -eq 1 ]; then
  # git add
  echo
  read -p 'Use `git add -f` on these files? Enter to continue ...'
  echo
  git add $FORCE $TEMPLATE_FILES
  echo
  if [ $GIT_COMMIT -eq 1 ]; then
    # git commit
    read -p 'Use `git commit` on these files? Enter to continue ...'
    # git commit -m "Initialize from template"
    git commit $TEMPLATE_FILES
    echo
  fi
fi

echo "- Finished applying template"
echo "  Template Used:     $TEMPLATE_NAME"
echo "  New Repository:    $REPODIR"
echo

[ $GIT_ADD -eq 1 ] && echo "  - Files have been added"
[ $GIT_COMMIT -eq 1 ] && echo "  - Files have been committed"

echo "Please use the following to push any changes:"
echo
echo "  pushd $REPODIR"
echo "  git push"
echo "  popd  # If you want to return to this directory ..."
echo