Skip to content

Instantly share code, notes, and snippets.

@sinbad
Created August 9, 2020 14:58
Show Gist options
  • Save sinbad/4bb771b916fa8facaf340af3fc49ee43 to your computer and use it in GitHub Desktop.
Save sinbad/4bb771b916fa8facaf340af3fc49ee43 to your computer and use it in GitHub Desktop.
My Gitea Backup & Restore Scripts
#!/bin/bash
# `gitea dump` doesn't currently back up LFS data as well, only git repos
# It primarily backs up the SQL DB, and also the config / logs
# We'll backup like this:
# * "gitea dump" to backup the DB and config etc
# * tar / bzip all the repos since they will be skipped
# * Not rotated because git data is immutable (normally) so has all data
# * rsync LFS data directly from /volume/docker/gitea/git/lfs
# * No need for rotation since all files are immutable
#
# This means our backup folder will contain:
# * /gitea_data.zip - containing the gitea data
# * /repositories/ - containing the bundles, structured owner/name.bundle
# * /lfs/ - containing all the direct LFS data
#
# Stop on errors
set -e
# Gitea config / SQL DB backup rotation
CONTAINER=gitea_server_1
# Backup dir from our perspective
HOST_BACKUP_DIR="/volume1/backups/gitea"
# Git repo dir from our perspective (it's outside container)
HOST_GIT_REPO_DIR="/volume1/docker/gitea/git/repositories"
# Git LFS dir from our perspective (it's outside container)
HOST_GIT_LFS_DIR="/volume1/docker/gitea/git/lfs"
# Where we work on things (host and container)
TEMP_DIR="/tmp"
GITEA_DATA_FILENAME="gitea_backup.zip"
HOST_BACKUP_FILE="$HOST_BACKUP_DIR/$GITEA_DATA_FILENAME"
# Back up to temp files then copy on success to prevent syncing incomplete/bad files
CONTAINER_BACKUP_FILE_TEMP="$TEMP_DIR/gitea_dump_temp.zip"
docker exec -u git -i $(docker ps -qf "name=$CONTAINER") bash -c "rm -f $CONTAINER_BACKUP_FILE_TEMP"
echo Backing up Gitea data to $HOST_BACKUP_FILE via $CONTAINER:$CONTAINER_BACKUP_FILE_TEMP
docker exec -u git -i $(docker ps -qf "name=$CONTAINER") bash -c "/app/gitea/gitea dump --skip-repository --skip-log --file $CONTAINER_BACKUP_FILE_TEMP"
# copy this into backup folder (in container)
docker cp $CONTAINER:$CONTAINER_BACKUP_FILE_TEMP $HOST_BACKUP_FILE
echo Backing up git repositories
# Git repos are in 2-level structure, owner/repository
# Again we MUST tar to a TEMP file and move into place when successful
GITREPO_BACKUP_FILE="$HOST_BACKUP_DIR/gitrepos_backup.tar.bz2"
GITREPO_BACKUP_FILE_TEMP=`mktemp -p $TEMP_DIR gitrepos_backup.tar.bz2.XXXXXX`
tar cjf $GITREPO_BACKUP_FILE_TEMP -C $HOST_GIT_REPO_DIR .
mv -f $GITREPO_BACKUP_FILE_TEMP $GITREPO_BACKUP_FILE
echo Backing up LFS data
# This syncs path/to/lfs to backup/dir/
# This will then be replicated directly to B2
# Yes this means we're storing LFS data twice but I prefer this to syncing to B2
# directly from the data dir, it makes the B2 sync simpler (just the whole folder)
rsync -rLptgo $HOST_GIT_LFS_DIR $HOST_BACKUP_DIR/
echo Gitea backup completed successfully
DROP DATABASE IF EXISTS gitea;
CREATE DATABASE gitea CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_unicode_ci';
GRANT ALL PRIVILEGES ON gitea.* TO 'gitea';
FLUSH PRIVILEGES;
exit
#!/bin/bash
usage () {
echo "Re-create a Gitea Docker container from a backup"
echo "Run this inside a docker-compose config folder!"
echo "If it's the test container (gitea_test) it'll be automated"
echo "If not, it will give you commands to run to complete the restore"
echo "Usage:"
echo " restore_gitea_container.sh [--unsafe] [--dry-run]"
echo ""
echo "Options:"
echo " --unsafe : Perform actions EVEN OUTSIDE gitea_test container"
echo " : BE CAREFUL with this, it will stomp your container"
echo " --dry-run : Only list actions, don't perform them"
echo ""
}
if [[ "$1" == "--help" ]]; then
usage
exit 0
fi
# MAKE SURE we're in the test dir
PWD=`pwd`
MANUAL_GUIDE=0
IS_TEST=1
if [[ ! -f "$PWD/docker-compose.yml" ]]; then
echo "You must run this inside a folder containing docker-compose.yml! Aborting"
exit 1
fi
if [[ ! "$PWD" == */docker-compose/gitea_test ]]; then
echo "HEY! You're not running this in docker-compose/gitea_test"
echo "So we're assuming this is a live instance and will not execute anything automatically"
echo "Instead, we'll print the commands you need to run."
MANUAL_GUIDE=1
IS_TEST=0
fi
while (( "$#" )); do
if [[ "$1" == "--dry-run" ]]; then
MANUAL_GUIDE=1
elif [[ "$1" == "--unsafe" ]]; then
while true; do
echo "Using --unsafe will destroy & re-create this container in all cases"
echo " It will probably get the ports wrong, be ready to edit app.ini afterwards!"
read -p "Are you SURE this is what you want? (y/n)" yn
case $yn in
[Yy]* ) break;;
[Nn]* ) exit;;
* ) echo "Please answer yes or no.";;
esac
done
MANUAL_GUIDE=0
else
if [[ "$1" == -* ]]; then
echo Unrecognised option $1
usage
exit 3
fi
fi
shift # $2 becomes $1..
done
# Generate all paths based on the last path component
# Root of all the host versions of what gets mapped to /data in container
DATAROOT="/volume1/docker"
# Backup source folder, contains gitea_backup.zip, gitrepos_backup.tar.bz2 & lfs
BACKUPSRC="/volume1/backups/gitea"
CONFIG=$(basename $PWD)
DATADIR="/volume1/docker/$CONFIG"
echo "Removing container"
# Use --all in case it's been run manually
if (( $MANUAL_GUIDE )); then
echo "Run this:"
echo " > docker-compose stop"
echo " > docker-compose rm"
else
docker-compose stop
docker-compose rm
fi
echo "Removing old Gitea data"
if (( $MANUAL_GUIDE )); then
echo "Run this:"
echo " > rm -r $DATADIR/*"
else
rm -r $DATADIR/*
fi
echo "Re-creating container"
if (( $MANUAL_GUIDE )); then
echo "Run this:"
echo " > docker-compose up --no-start"
else
docker-compose up --no-start
fi
# We now need to bring up the database server to restore the MySQL data
# Bring it up early so that it's got time to start while we do the data copying
echo "Bringing up database to restore"
if (( $MANUAL_GUIDE )); then
echo "Run this:"
echo " > docker-compose start db"
else
docker-compose start db
fi
# Run the restore script to get back the contents of docker/gitea data folder
echo Restoring Gitea, Git and Git-LFS data
# Copy SQL to MySQL's folder (from host perspective)
MYSQLDATADIR="/volume1/docker/${CONFIG}_db"
MYSQLFILE=`mktemp -t gitea-db.sql.XXXXXX`
if (( $MANUAL_GUIDE )); then
echo "Run this:"
echo " > ../../backups/restore_gitea_data.sh $BACKUPSRC $DATADIR $MYSQLFILE"
else
../../backups/restore_gitea_data.sh $BACKUPSRC $DATADIR $MYSQLFILE
fi
# Restore DB
# We need to make sure it's up, it can take a little time before connections
# are allowed
MYSQL_ATTEMPTS=3
while [[ $MYSQL_ATTEMPTS -gt 0 ]] ; do
echo "Testing if MySQL is up"
let MYSQL_ATTEMPTS--
if docker-compose exec db mysqladmin -uroot -pDB_ROOT_PASSWORD status; then
break
fi
sleep 2
done
# Our docker container creates the gitea user and gitea DB in all cases
# Drop all tables first
# Can't just pipe in data to docker-compose exec because bug https://github.com/docker/compose/issues/3352
# Fixed but not in the Synology version
# We can use main docker but need to parse out the ID for alias 'db'
DB_DOCKER_ID=$(docker-compose ps -q db)
if (( $MANUAL_GUIDE )); then
echo "Run this:"
echo " > docker exec -i $DB_DOCKER_ID mysql -uroot -pDB_ROOT_PASSWORD < ../../sql/reset-gitea-mysql.sql"
echo "> docker exec -i $DB_DOCKER_ID mysql -ugitea -pDB_GITEA_PASSWORD gitea < $MYSQLFILE"
else
echo "Restoring database....be patient!"
docker exec -i $DB_DOCKER_ID mysql -uroot -pDB_ROOT_PASSWORD < ../../sql/reset-gitea-mysql.sql
docker exec -i $DB_DOCKER_ID mysql -ugitea -pDB_GITEA_PASSWORD gitea < $MYSQLFILE
fi
# Clean up
rm -f $MYSQLFILE
# Need to modify app.ini to change ports on URLs
if (( $IS_TEST )); then
echo Fixing up ports
if (( $MANUAL_GUIDE )); then
echo "You need to edit $DATADIR/gitea/conf.app.ini, change:"
echo " - ROOT_URL = https://git.yourserver.com:9000/"
echo " + ROOT_URL = https://git.yourserver.com:10000/"
echo " - SSH_PORT = 9022"
echo " - SSH_PORT = 10022"
else
sed -i "s/\.com:9000/.com:10000" $DATADIR/gitea/conf/app.ini
sed -i "s/9022/10022" $DATADIR/gitea/conf/app.ini
fi
else
echo "Since this isn't the test environment, you'll need to check $DATADIR/gitea/conf.app.ini manually!"
fi
# This will have created the missing ssh folder w/ server config etc
echo "Restoration complete"
echo "Restarting Server"
if (( $MANUAL_GUIDE )); then
echo "Run this:"
echo " > docker-compose up -d"
else
docker-compose up -d
fi
echo "NOTE: SSH keys will likely not work if you're restoring to another server"
echo " Users will probably have to remove & re-add their keys in Settings"
#!/bin/bash
# See backup_gitea.sh
# Source to restore should include:
# - gitea_backup.zip
# - gitrepos_backup.tar.bz2
# - lfs/*/*/*
# Exit on error
set -e
usage () {
echo "Script for restoring Gitea data from a backup. Will REPLACE existing data!!"
echo "You probably DON'T WANT THIS SCRIPT directly, it only gets the data files back"
echo "Look at restore_gitea_container.sh for a fully automated Docker container & DB restore"
echo "Usage:"
echo " restore_gitea_data.sh [--dry-run] <source_dir> <dest_dir> <sql_file_dest>"
echo ""
echo "Params:"
echo " source_dir : Directory containing Gitea backup"
echo " dest_dir : Directory to write data to (host, mapped to /data in container)"
echo " sql_file_dest : Full file path of where to place gitea-db.sql from backup"
echo "Options:"
echo " --dry-run : Don't actually perform actions, just report"
echo ""
}
if [[ "$1" == "--help" ]]; then
usage
exit 0
fi
DRYRUN=0
SOURCE=""
DATADIR=""
SQLDEST=""
USER_UID=1000
GROUP_GID=1000
while (( "$#" )); do
if [[ "$1" == "--dry-run" ]]; then
DRYRUN=1
else
if [[ "$1" == -* ]]; then
echo Unrecognised option $1
usage
exit 3
# Populate positional args
elif [[ "$SOURCE" == "" ]]; then
SOURCE=$1
elif [[ "$DATADIR" == "" ]]; then
DATADIR=$1
else
SQLDEST=$1
fi
fi
shift # $2 becomes $1..
done
if [[ "$SOURCE" == "" ]]; then
echo "Required: source folder"
usage
exit 3
fi
if [[ "$DATADIR" == "" ]]; then
echo "Required: destination data dir"
usage
exit 3
fi
echo Checking required files exist in $SOURCE
if [[ ! -f "$SOURCE/gitea_backup.zip" ]]; then
echo "ERROR: Missing file in restore $SOURCE/gitea_backup.zip"
exit 5
fi
if [[ ! -f "$SOURCE/gitrepos_backup.tar.bz2" ]]; then
echo "ERROR: Missing file in restore $SOURCE/gitrepos_backup.tar.bz2"
exit 5
fi
if [[ ! -d "$SOURCE/lfs" ]]; then
echo "ERROR: Missing directory in restore $SOURCE/lfs"
exit 5
fi
# The only thing we can't restore is the gitea/ssh folder, which contains the
# server SSH keys. We leave that for Gitea to re-create on first start
echo Checking container data
if [[ ! -d "$DATADIR/gitea" ]]; then
if (( $DRYRUN )); then
echo "Would have created $DATADIR/gitea"
else
echo "Creating $DATADIR/gitea"
mkdir -p $DATADIR/gitea
chown -R $USER_UID:$GROUP_GID $DATADIR/gitea
chmod -R u+rwX,go+rX,go-w $DATADIR/gitea
fi
fi
if [[ ! -d "$DATADIR/git/repositories" ]]; then
if (( $DRYRUN )); then
echo "Would have created $DATADIR/git/repositories"
else
echo "Creating $DATADIR/git/repositories"
mkdir -p $DATADIR/git/repositories
chown -R $USER_UID:$GROUP_GID $DATADIR/git/repositories
chmod -R u+rwX,go+rX,go-w $DATADIR/git/repositories
fi
fi
if [[ ! -d "$DATADIR/git/lfs" ]]; then
if (( $DRYRUN )); then
echo "Would have created $DATADIR/git/lfs"
else
echo "Creating $DATADIR/git/lfs"
mkdir -p $DATADIR/git/lfs
chown -R $USER_UID:$GROUP_GID $DATADIR/git/lfs
chmod -R u+rwX,go+rX,go-w $DATADIR/git/lfs
fi
fi
GITEA_DATA_FILE="$SOURCE/gitea_backup.zip"
echo "** Step 1 of 3: Gitea data START **"
echo "Copying Gitea files back from $GITEA_DATA_FILE"
# gitea_backup.zip contains:
# gitea-db.sql - database dump in SQL form
# app.ini - same as custom/conf/app.ini
#
# custom/ - All subfolders go back to data folder
# conf/
# log/
# queues/
# gitea.db - The actual SQLite DB file, seems the same?
# indexers/
# sessions/
# avatars/
# data/ - Again, all subfolders go back to data, same as custom??
# conf/
# log/
# queues/
# gitea.db - Actual sqlite data file
# indexers/
# sessions/
# avatars/
# log/ - Log files, we don't need these (will remove from backup script)
# So there seems to be a lot of duplication in this dump
# Even when I don't "gitea dump" with -C it still creates custom/ and data/
# I think we only want the /data dir
# Extract all to temp then copy
TEMPDIR=`mktemp -d`
# protect! because we're going to rm -rf this later
if [[ ! "$TEMPDIR" == /tmp/* ]]; then
echo Error: expected mktemp to give us a dir in /tmp, being careful & aborting
fi
echo "Extracting archive..."
# We have to use 7z because that comes pre-installed on Synology but unzip doesn't
# Send to /dev/null as 7z writes a bunch of crap that doesn't translate well to some terminals
7z x -o$TEMPDIR $GITEA_DATA_FILE > /dev/null 2>&1
# Unfortunately 7z doesn't restore file permissions / ownership
# Docker Gitea has everything owned by 1000:1000
echo "Fixing permissions"
chown -R $USER_UID:$GROUP_GID $TEMPDIR
# And permissions are 755/644 for dirs / files
# The capital X only sets x bit on dirs, not files, which gives us 755/644
chmod -R u+rwX,go+rX,go-w $TEMPDIR
GITEA_DEST="$DATADIR/gitea"
if (( $DRYRUN )); then
echo "Would have copied data directory $TEMPDIR/data to $GITEA_DEST"
echo "This was the structure:"
ls -la $TEMPDIR/data
else
echo "Copying data directory $TEMPDIR/data to $GITEA_DEST"
# Note -p is ESSENTIAL to preserve ownership
cp -p -R -f $TEMPDIR/data/* $GITEA_DEST/
fi
if [[ ! "$SQLDEST" == "" ]]; then
if (( $DRYRUN )); then
echo "Would have copied $TEMPDIR/gitea-db.sql to $SQLDEST"
else
echo "Copying $TEMPDIR/gitea-db.sql to $SQLDEST"
# Note -p is ESSENTIAL to preserve ownership
cp -p -f $TEMPDIR/gitea-db.sql $SQLDEST
fi
fi
rm -rf $TEMPDIR
echo "** Step 1 of 3: Gitea data DONE **"
# Now do repositories
# tar preserves owner/permissions, and we tarred relative to data/git/repositories
# so we can do this direct
# remove all existing data so it's clean
GIT_REPO_FILE="$SOURCE/gitrepos_backup.tar.bz2"
GIT_REPO_DEST="$DATADIR/git/repositories"
echo "** Step 2 of 3: Git repository data START **"
echo "Restoring git repository data"
if (( $DRYRUN )); then
echo "Would have deleted $GIT_REPO_DEST/*"
echo "Would have extracted $GIT_REPO_FILE to $GIT_REPO_DEST"
echo "Repositories were: "
# equivalent of depth = 2
tar --exclude="*/*/*/*" -tf $GIT_REPO_FILE
else
echo "Cleaning existing repo data"
rm -rf $GIT_REPO_DEST/*
echo "Extracting $GIT_REPO_FILE to $GIT_REPO_DEST"
tar -xf $GIT_REPO_FILE -C $GIT_REPO_DEST
fi
echo "Git repositories done"
echo "** Step 2 of 3: Git repository data DONE **"
echo "** Step 3 of 3: Git-LFS data START **"
echo "Restoring Git-LFS data"
GIT_LFS_SRC="$SOURCE/lfs"
GIT_LFS_DEST="$DATADIR/git/lfs"
# There is no reason to delete LFS data since it's all immutable
# Instead rsync back like we did during backup
if (( $DRYRUN )); then
echo "Would have synced LFS data from $GIT_LFS_SRC to $GIT_LFS_DEST"
else
echo "Syncing LFS data from $GIT_LFS_SRC to $GIT_LFS_DEST"
rsync -rLptgo $GIT_LFS_SRC/* $GIT_LFS_DEST/
echo "Fixing LFS permissions"
chown -R $USER_UID:$GROUP_GID $GIT_LFS_DEST
# And permissions are 755/644 for dirs / files
# The capital X only sets x bit on dirs, not files, which gives us 755/644
chmod -R u+rwX,go+rX,go-w $GIT_LFS_DEST
fi
echo "** Step 3 of 3: Git-LFS data DONE **"
echo "Gitea data restored successfully"
@wlcx
Copy link

wlcx commented Sep 14, 2022

FWIW, given this is high up in search results for "gitea backup", it looks like gitea dump does now include LFS data.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment