Skip to content

Instantly share code, notes, and snippets.

@drakonstein
Last active May 19, 2023 02:50
Show Gist options
  • Save drakonstein/df99a1becd16bc96eb57a8ab230611a0 to your computer and use it in GitHub Desktop.
Save drakonstein/df99a1becd16bc96eb57a8ab230611a0 to your computer and use it in GitHub Desktop.
Script to automate snapshots for all RBDs in a pool based on key/value settings on RBDs
#!/bin/bash
[[ "$1" == *list* ]] && list=true || list=false
[[ "$1" == *force* ]] && force=true || force=false
ENABLED=true
# List of features to ensure are enabled on each RBD
FEATURES='layering exclusive-lock object-map fast-diff'
# Prefix for the metadata keys on RBDs for the backup snapshots
KEY_PREFIX=backsnap
# How many hours between backups. Anything over 24 will back up once/day.
FREQUENCY=12
# Settings are to indicate when to start only keeping that type of backup around
CLEANUP='6 months'
MONTHLY='2 months'
WEEKLY='14 days'
DAILY='24 hours'
now=$(date +%s)
time=$(date -d@$now +%Y%m%d-%H:%M)
hour=$(date -d@$now +%_H)
for pool in $(ceph -f json osd lspools | jq -r '.[].poolname'); do
pool_settings=$(ceph -f json osd pool application get $pool)
# Skip pools not configured as an RBD
[[ $(echo "$pool_settings" | jq -r '.rbd') == null ]] && continue
# Set variables for each RBD pool
pool_enabled=$(echo "$pool_settings" | jq -r .rbd.${KEY_PREFIX}_enabled)
[[ "$pool_enabled" == null ]] && pool_enabled=$ENABLED
[[ $pool_enabled || $force ]] || continue
pool_frequency=$(echo "$pool_settings" | jq -r .rbd.${KEY_PREFIX}_frequency)
[[ "$pool_frequency" == null ]] && pool_frequency=$FREQUENCY
pool_cleanup=$(echo "$pool_settings" | jq -r .rbd.${KEY_PREFIX}_cleanup)
[[ "$pool_cleanup" == null ]] && pool_cleanup=$CLEANUP
pool_monthly=$(echo "$pool_settings" | jq -r .rbd.${KEY_PREFIX}_monthly)
[[ "$pool_monthly" == null ]] && pool_monthly=$MONTHLY
pool_weekly=$(echo "$pool_settings" | jq -r .rbd.${KEY_PREFIX}_weekly)
[[ "$pool_weekly" == null ]] && pool_weekly=$WEEKLY
pool_daily=$(echo "$pool_settings" | jq -r .rbd.${KEY_PREFIX}_daily)
[[ "$pool_daily" == null ]] && pool_daily=$DAILY
# Spaces, commas, etc aren't allowed in the value of pool settings so we're using periods
pool_features=$(echo "$pool_settings" | jq -r .rbd.${KEY_PREFIX}_features | tr '.' ' ')
[[ "$pool_features" == null ]] && pool_features=$FEATURES
for rbd in $(rbd -p $pool ls); do
snapshots=$(rbd --format json -p $pool snap ls $rbd | jq -r '.[].name')
if $list; then
echo '================================================='
echo "$pool/$rbd"
echo "$snapshots"
continue
fi
# Set variables for each RBD
rebuild_object_map=false
rbd_settings=$(rbd --format json -p $pool image-meta list $rbd)
rbd_enabled=$(echo "$rbd_settings" | jq -r .${KEY_PREFIX}_enabled)
[[ -z "$rbd_enabled" || "$rbd_enabled" == null ]] && rbd_enabled=$pool_enabled
[[ $rbd_enabled || $force ]] || continue
rbd_frequency=$(echo "$rbd_settings" | jq -r .${KEY_PREFIX}_frequency)
[[ -z "$rbd_frequency" || "$rbd_frequency" == null ]] && rbd_frequency=$pool_frequency
rbd_cleanup=$(echo "$rbd_settings" | jq -r .${KEY_PREFIX}_cleanup)
[[ -z "$rbd_cleanup" || "$rbd_cleanup" == null ]] && rbd_cleanup=$pool_cleanup
rbd_monthly=$(echo "$rbd_settings" | jq -r .${KEY_PREFIX}_monthly)
[[ -z "$rbd_monthly" || "$rbd_monthly" == null ]] && rbd_monthly=$pool_monthly
rbd_weekly=$(echo "$rbd_settings" | jq -r .${KEY_PREFIX}_weekly)
[[ -z "$rbd_weekly" || "$rbd_weekly" == null ]] && rbd_weekly=$pool_weekly
rbd_daily=$(echo "$rbd_settings" | jq -r .${KEY_PREFIX}_daily)
[[ -z "$rbd_daily" || "$rbd_daily" == null ]] && rbd_daily=$pool_daily
# Some features need to be enabled in a specific order.
# Attempt to update the features until all of them succeed.
update_features=true
count=0
while $update_features && (( count < 5 )); do
rbd_features=$(rbd --format json -p $pool info $rbd | jq -r '.features | .[]')
update_features=false
(( count += 1 ))
for feature in $pool_features; do
echo "$rbd_features" | grep -q "^$feature$" || {
rbd -p $pool feature enable $rbd $feature || {
update_features=true
echo "Failed to enable feature $feature on $pool/$rbd"
}
[[ "$feature" == "object-map" ]] && rebuild_object_map=true
}
done
done
$rebuild_object_map && rbd -p $pool object-map rebuild $rbd > /dev/null 2>&1
# Skip backup run if ${KEY_PREFIX}_enabled is false unless force is specified
if $rbd_enabled || $force; then
# Ony run backups where the frequency matches up unless force is specified
if (( hour % rbd_frequency == 0 )) || $force; then
# Only run backups once/hour unless force is specified
if ! echo "$snapshots" | grep -q "^${time::11}:" || $force; then
# If we don't successfully create a new snapshot let's not do anything else for this RBD
echo "$pool/$rbd@$time Creating snapshot"
rbd -p $pool snap create $rbd@$time || continue
else
continue
fi
else
continue
fi
else
continue
fi
# Skip cleanup if this is the last snapshot
(( $(echo "$snapshots" | wc -l) == 1 )) && continue
cleanup=$(date -d@$now -d "-$rbd_cleanup" +%Y%m%d%H%M)
monthly=$(date -d@$now -d "-$rbd_monthly" +%Y%m%d%H%M)
weekly=$(date -d@$now -d "-$rbd_weekly" +%Y%m%d%H%M)
daily=$(date -d@$now -d "-$rbd_daily" +%Y%m%d%H%M)
for snapshot in $snapshots; do
$rebuild_object_map && rbd -p $pool object-map rebuild $rbd@$snapshot > /dev/null 2>&1
# Don't clean up if this was a forced run, but allow the object_map to be fixed first
$force && continue
# Only clean up snapshots made using our formatting
echo $snapshot | grep -qE '^[[:digit:]]{8}-[[:digit:]]{2}:[[:digit:]]{2}$' || continue
snaptime=$(echo $snapshot | tr -d ':-')
if (( cleanup > snaptime )); then
echo "$pool/$rbd@$snapshot Deleting snapshot"
rbd -p $pool snap rm $rbd@$snapshot
continue
elif (( monthly > snaptime )); then
# shellcheck disable=SC2001
snapday=$(echo ${snaptime:6:2} | sed 's/^0//')
first_sunday=$(ncal -m ${snaptime:4:2} ${snaptime:0:4} | awk '/Su/ {print $2}')
if (( snapday != first_sunday )); then
echo "$pool/$rbd@$snapshot Deleting snapshot"
rbd -p $pool snap rm $rbd@$snapshot
fi
continue
elif (( weekly > snaptime )); then
snapday=$(date -d ${snaptime:0:8} +%A)
if [[ "$snapday" != "Sunday" ]]; then
echo "$pool/$rbd@$snapshot Deleting snapshot"
rbd -p $pool snap rm $rbd@$snapshot
fi
continue
elif (( daily > snaptime )); then
snaphour=${snaptime:8:2}
if [[ "$snaphour" != "00" ]]; then
echo "$pool/$rbd@$snapshot Deleting snapshot"
rbd -p $pool snap rm $rbd@$snapshot
fi
continue
fi
done &
done
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment