Skip to content

Instantly share code, notes, and snippets.

@jessiel-hacke
Last active October 22, 2019 19:46
Show Gist options
  • Save jessiel-hacke/a8eaa8857f738e66b735a9d33a93bffc to your computer and use it in GitHub Desktop.
Save jessiel-hacke/a8eaa8857f738e66b735a9d33a93bffc to your computer and use it in GitHub Desktop.
PG Backup
##############################
## POSTGRESQL BACKUP CONFIG ##
##############################
# Optional system user to run backups as. If the user the script is running as doesn't match this
# the script terminates. Leave blank to skip check.
BACKUP_USER=postgres
# Optional hostname to adhere to pg_hba policies. Will default to "localhost" if none specified.
HOSTNAME=localhost
# Optional username to connect to database as. Will default to "postgres" if none specified.
USERNAME=postgres
# This dir will be created if it doesn't exist. This must be writable by the user the script is
# running as.
BACKUP_DIR=backups/database/postgresql/
# List of strings to match against in database name, separated by space or comma, for which we only
# wish to keep a backup of the schema, not the data. Any database names which contain any of these
# values will be considered candidates. (e.g. "system_log" will match "dev_system_log_2010-01")
SCHEMA_ONLY_LIST=
# List of strings to match against in database name, separated by space or comma
#
FULL_BACKUP_LIST="new_db"
# Will produce a custom-format backup if set to "yes"
ENABLE_CUSTOM_BACKUPS=no
# Will produce a gzipped plain-format backup if set to "yes"
ENABLE_PLAIN_BACKUPS=yes
# Will produce gzipped sql file containing the cluster globals, like users and passwords, if set to "yes"
ENABLE_GLOBALS_BACKUPS=no
#### SETTINGS FOR ROTATED BACKUPS ####
## Daily Backups ##
ENABLE_DAILY_BACKUP=yes # Enable daily backups (default: no)
DAYS_TO_KEEP=2 # Number of days to keep daily backups (default: 7)
## Weekly Backups ##
ENABLE_WEEKLY_BACKUP=no # Enable weekly backups (default: no)
WEEKS_TO_KEEP=4 # How many weeks to keep weekly backups (default: 4)
DAY_OF_WEEK_TO_KEEP=5 # Which day to take the weekly backup from (default: 5, options: 1-7 = Monday-Sunday)
## Monthly Backups ##
ENABLE_MONTHLY_BACKUP=no # Enable monthly backups (default: no)
DAY_OF_MONTH_TO_KEEP=01 # Which day to take the monthly backup from (default: 01 = first of the month)
#!/bin/bash
###########################
########### LOG ###########
###########################
function log()
{
MESSAGE=$1
if [ -z $LOG_FILE ]; then
LOG_FILE="./backup.log"
fi
echo "`date '+%d-%m-%Y %H:%M:%S'` $MESSAGE" >> $LOG_FILE
}
###########################
####### LOAD CONFIG #######
###########################
while [ $# -gt 0 ]; do
case $1 in
-c)
CONFIG_FILE_PATH="$2"
shift 2
;;
*)
log "Unknown Option"
exit 2
;;
esac
done
if [ -z $CONFIG_FILE_PATH ] ; then
SCRIPTPATH=$(pwd -P)
CONFIG_FILE_PATH="${SCRIPTPATH}/backup.config"
log "Loaded config file from ${CONFIG_FILE_PATH}"
fi
if [ ! -r ${CONFIG_FILE_PATH} ] ; then
log "Could not load config file from ${CONFIG_FILE_PATH}"
exit 1
fi
source "${CONFIG_FILE_PATH}"
###########################
#### PRE-BACKUP CHECKS ####
###########################
# Make sure we're running as the required backup user
if [ "$BACKUP_USER" != "" -a "$(id -un)" != "$BACKUP_USER" ] ; then
log "This script must be run as $BACKUP_USER. Exiting."
exit 1
fi
###########################
### INITIALISE DEFAULTS ###
###########################
if [ -z $HOSTNAME -a -z $USERNAME ]; then
DB_CONFIG=`-h "$HOSTNAME" -U "$USERNAME"`
else
DB_CONFIG=""
fi
###########################
#### START THE BACKUPS ####
###########################
function perform_backups()
{
SUFFIX=$1
FINAL_BACKUP_DIR=$BACKUP_DIR"`date +\%Y-\%m-\%d`$SUFFIX/"
log "Making backup directory in $FINAL_BACKUP_DIR"
if ! mkdir -p $FINAL_BACKUP_DIR; then
log "Cannot create backup directory in $FINAL_BACKUP_DIR. Go and fix it!"
exit 1;
fi;
#######################
### GLOBALS BACKUPS ###
#######################
log "Globals Backup: Checking Configuration..."
log "--------------------------------------------"
if [ $ENABLE_GLOBALS_BACKUPS = "yes" ]
then
log "Globals Backup: Started!"
if ! pg_dumpall -g $DB_CONFIG | gzip > $FINAL_BACKUP_DIR"globals".sql.gz.in_progress; then
log "[!!ERROR!!] Failed to produce globals backup"
else
mv $FINAL_BACKUP_DIR"globals".sql.gz.in_progress $FINAL_BACKUP_DIR"globals".sql.gz
log "Globals Backup: Backup compressed and saved"
fi
else
log "Globals Backup: Not Enabled!"
fi
log "Globals Backup: DONE!"
###########################
### SCHEMA-ONLY BACKUPS ###
###########################
log "Schema-Only Backup: Checking Configuration..."
for SCHEMA_ONLY_DB in ${SCHEMA_ONLY_LIST//,/ }
do
SCHEMA_ONLY_CLAUSE="$SCHEMA_ONLY_CLAUSE or datname ~ '$SCHEMA_ONLY_DB'"
done
SCHEMA_ONLY_QUERY="select datname from pg_database where false $SCHEMA_ONLY_CLAUSE order by datname;"
log "Schema-Only Backup: Started!"
log "--------------------------------------------"
SCHEMA_ONLY_DB_LIST=`psql $DB_CONFIG -At -c "$SCHEMA_ONLY_QUERY" postgres`
log "Schema-Only Backup: The following databases were matched:$SCHEMA_ONLY_DB_LIST"
for DATABASE in $SCHEMA_ONLY_DB_LIST
do
log "Schema-Only Backup: Started for DB :: $DATABASE ::"
if ! pg_dump -Fp -s $DB_CONFIG "$DATABASE" | gzip > $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress; then
log "Schema-Only Backup: [!!ERROR!!] Failed to backup database :: $DATABASE ::"
else
mv $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz
log "Schema-Only Backup: Backup saved for DB :: $DATABASE ::"
fi
done
log "Schema-Only Backup: DONE!"
###########################
###### FULL BACKUPS #######
###########################
log "Full Backup: Checking Configuration..."
log "${FULL_BACKUP_LIST//,/ }"
for FULL_BACKUP_DB in ${FULL_BACKUP_LIST//,/ }
do
FULL_BACKUP_CLAUSE="$FULL_BACKUP_CLAUSE and datname ~ '$FULL_BACKUP_DB'"
done
FULL_BACKUP_QUERY="select datname from pg_database where not datistemplate and datallowconn $FULL_BACKUP_CLAUSE order by datname;"
log "Full Backup: Started!"
log "--------------------------------------------"
DATABASES=`psql $DB_CONFIG -At -c "$FULL_BACKUP_QUERY"`
log "$DATABASES"
log "Full Backup: The following databases were matched: $DATABASES"
for DATABASE in $DATABASES
do
if [ $ENABLE_PLAIN_BACKUPS = "yes" ]
then
log "Full Backup: Plain backup of $DATABASE"
if ! pg_dump -Fp $DB_CONFIG "$DATABASE" | gzip > $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress; then
log "Full Backup: [!!ERROR!!] Failed to produce plain backup for database $DATABASE"
else
mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
log "Full Backup: Succesfully produced plain backup for database :: $DATABASE ::"
fi
fi
if [ $ENABLE_CUSTOM_BACKUPS = "yes" ]
then
log "Full Backup: Custom backup of $DATABASE"
if ! pg_dump -Fc $DB_CONFIG "$DATABASE" -f $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress; then
log "Full Backup: [!!ERROR!!] Failed to produce custom backup for database $DATABASE"
else
mv $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress $FINAL_BACKUP_DIR"$DATABASE".custom
log "Full Backup: Succesfully produced custom backup for database :: $DATABASE ::"
fi
fi
done
log "Full Backup: DONE!"
}
# MONTHLY BACKUPS
DAY_OF_MONTH=`date +%d`
if [ $ENABLE_MONTHLY_BACKUP = yes -a $DAY_OF_MONTH = $DAY_OF_MONTH_TO_KEEP ];
then
# Delete all expired monthly directories
find $BACKUP_DIR -maxdepth 1 -name "*-monthly" -exec rm -rf '{}' ';'
log "Monthly Backup: Started!"
perform_backups "-monthly"
log "Monthly Backup: DONE!"
exit 0;
else
log "Monthly Backup: Skipped!"
fi
# WEEKLY BACKUPS
DAY_OF_WEEK=`date +%u` #1-7 (Monday-Sunday)
EXPIRED_DAYS=`expr $((($WEEKS_TO_KEEP * 7) + 1))`
if [ $ENABLE_WEEKLY_BACKUP = yes -a $DAY_OF_WEEK = $DAY_OF_WEEK_TO_KEEP ];
then
# Delete all expired weekly directories
find $BACKUP_DIR -maxdepth 1 -mtime $EXPIRED_DAYS -name "*-weekly" -exec rm -rf '{}' ';'
log "Weekly Backup: Started!"
perform_backups "-weekly"
log "Weekly Backup: DONE!"
exit 0;
else
log "Weekly Backup: Skipped!"
fi
# DAILY BACKUPS
# Delete daily backups 7 days old or more
if [ $ENABLE_DAILY_BACKUP = yes ];
then
find $BACKUP_DIR -maxdepth 1 -mtime $DAYS_TO_KEEP -name "*-daily" -exec rm -rf '{}' ';'
log "Daily Backup: Started!"
perform_backups "-daily"
log "Daily Backup: DONE!"
exit 0;
else
log "Daily Backup: Skipped!"
fi
log "All Backups were completed!"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment