Script to automate snapshots for all RBDs in a pool based on key/value settings on RBDs

rbd_snapshots.sh

#!/bin/bash

[[ "$1" == *list* ]] && list=true || list=false
[[ "$1" == *force* ]] && force=true || force=false
ENABLED=true
# List of features to ensure are enabled on each RBD
FEATURES='layering exclusive-lock object-map fast-diff'
# Prefix for the metadata keys on RBDs for the backup snapshots
KEY_PREFIX=backsnap
# How many hours between backups. Anything over 24 will back up once/day.
FREQUENCY=12
# Settings are to indicate when to start only keeping that type of backup around
CLEANUP='6 months'
MONTHLY='2 months'
WEEKLY='14 days'
DAILY='24 hours'
now=$(date +%s)
time=$(date -d@$now +%Y%m%d-%H:%M)
hour=$(date -d@$now +%_H)


for pool in $(ceph -f json osd lspools | jq -r '.[].poolname'); do
  pool_settings=$(ceph -f json osd pool application get $pool)
  # Skip pools not configured as an RBD
  [[ $(echo "$pool_settings" | jq -r '.rbd') == null ]] && continue

  # Set variables for each RBD pool
  pool_enabled=$(echo "$pool_settings" | jq -r .rbd.${KEY_PREFIX}_enabled)
  [[ "$pool_enabled" == null ]] && pool_enabled=$ENABLED
  [[ $pool_enabled || $force ]] || continue
  pool_frequency=$(echo "$pool_settings" | jq -r .rbd.${KEY_PREFIX}_frequency)
  [[ "$pool_frequency" == null ]] && pool_frequency=$FREQUENCY
  pool_cleanup=$(echo "$pool_settings" | jq -r .rbd.${KEY_PREFIX}_cleanup)
  [[ "$pool_cleanup" == null ]] && pool_cleanup=$CLEANUP
  pool_monthly=$(echo "$pool_settings" | jq -r .rbd.${KEY_PREFIX}_monthly)
  [[ "$pool_monthly" == null ]] && pool_monthly=$MONTHLY
  pool_weekly=$(echo "$pool_settings" | jq -r .rbd.${KEY_PREFIX}_weekly)
  [[ "$pool_weekly" == null ]] && pool_weekly=$WEEKLY
  pool_daily=$(echo "$pool_settings" | jq -r .rbd.${KEY_PREFIX}_daily)
  [[ "$pool_daily" == null ]] && pool_daily=$DAILY
  # Spaces, commas, etc aren't allowed in the value of pool settings so we're using periods
  pool_features=$(echo "$pool_settings" | jq -r .rbd.${KEY_PREFIX}_features | tr '.' ' ')
  [[ "$pool_features" == null ]] && pool_features=$FEATURES


  for rbd in $(rbd -p $pool ls); do
    snapshots=$(rbd --format json -p $pool snap ls $rbd | jq -r '.[].name')
    if $list; then
      echo '================================================='
      echo "$pool/$rbd"
      echo "$snapshots"
      continue
    fi

    # Set variables for each RBD
    rebuild_object_map=false
    rbd_settings=$(rbd --format json -p $pool image-meta list $rbd)
    rbd_enabled=$(echo "$rbd_settings" | jq -r .${KEY_PREFIX}_enabled)
    [[ -z "$rbd_enabled" || "$rbd_enabled" == null ]] && rbd_enabled=$pool_enabled
    [[ $rbd_enabled || $force ]] || continue
    rbd_frequency=$(echo "$rbd_settings" | jq -r .${KEY_PREFIX}_frequency)
    [[ -z "$rbd_frequency" || "$rbd_frequency" == null ]] && rbd_frequency=$pool_frequency
    rbd_cleanup=$(echo "$rbd_settings" | jq -r .${KEY_PREFIX}_cleanup)
    [[ -z "$rbd_cleanup" || "$rbd_cleanup" == null ]] && rbd_cleanup=$pool_cleanup
    rbd_monthly=$(echo "$rbd_settings" | jq -r .${KEY_PREFIX}_monthly)
    [[ -z "$rbd_monthly" || "$rbd_monthly" == null ]] && rbd_monthly=$pool_monthly
    rbd_weekly=$(echo "$rbd_settings" | jq -r .${KEY_PREFIX}_weekly)
    [[ -z "$rbd_weekly" || "$rbd_weekly" == null ]] && rbd_weekly=$pool_weekly
    rbd_daily=$(echo "$rbd_settings" | jq -r .${KEY_PREFIX}_daily)
    [[ -z "$rbd_daily" || "$rbd_daily" == null ]] && rbd_daily=$pool_daily

    # Some features need to be enabled in a specific order.
    # Attempt to update the features until all of them succeed.
    update_features=true
    count=0
    while $update_features && (( count < 5 )); do
      rbd_features=$(rbd --format json -p $pool info $rbd | jq -r '.features | .[]')
      update_features=false
      (( count += 1 ))
      for feature in $pool_features; do
        echo "$rbd_features" | grep -q "^$feature$" || {
          rbd -p $pool feature enable $rbd $feature || {
            update_features=true
            echo "Failed to enable feature $feature on $pool/$rbd"
          }
          [[ "$feature" == "object-map" ]] && rebuild_object_map=true
        }
      done
    done
    $rebuild_object_map && rbd -p $pool object-map rebuild $rbd > /dev/null 2>&1

    # Skip backup run if ${KEY_PREFIX}_enabled is false unless force is specified
    if $rbd_enabled || $force; then
      # Ony run backups where the frequency matches up unless force is specified
      if (( hour % rbd_frequency == 0 )) || $force; then
        # Only run backups once/hour unless force is specified
        if ! echo "$snapshots" | grep -q "^${time::11}:" || $force; then
          # If we don't successfully create a new snapshot let's not do anything else for this RBD
          echo "$pool/$rbd@$time Creating snapshot"
          rbd -p $pool snap create $rbd@$time || continue
        else
          continue
        fi
      else
        continue
      fi
    else
      continue
    fi

    # Skip cleanup if this is the last snapshot
    (( $(echo "$snapshots" | wc -l) == 1 )) && continue
    cleanup=$(date -d@$now -d "-$rbd_cleanup" +%Y%m%d%H%M)
    monthly=$(date -d@$now -d "-$rbd_monthly" +%Y%m%d%H%M)
    weekly=$(date -d@$now -d "-$rbd_weekly" +%Y%m%d%H%M)
    daily=$(date -d@$now -d "-$rbd_daily" +%Y%m%d%H%M)
    for snapshot in $snapshots; do
      $rebuild_object_map && rbd -p $pool object-map rebuild $rbd@$snapshot > /dev/null 2>&1
      # Don't clean up if this was a forced run, but allow the object_map to be fixed first
      $force && continue

      # Only clean up snapshots made using our formatting
      echo $snapshot | grep -qE '^[[:digit:]]{8}-[[:digit:]]{2}:[[:digit:]]{2}$' || continue
      snaptime=$(echo $snapshot | tr -d ':-')

      if (( cleanup > snaptime )); then
        echo "$pool/$rbd@$snapshot Deleting snapshot"
        rbd -p $pool snap rm $rbd@$snapshot
        continue
      elif (( monthly > snaptime )); then
        # shellcheck disable=SC2001
        snapday=$(echo ${snaptime:6:2} | sed 's/^0//')
        first_sunday=$(ncal -m ${snaptime:4:2} ${snaptime:0:4} | awk '/Su/ {print $2}')
        if (( snapday != first_sunday )); then
          echo "$pool/$rbd@$snapshot Deleting snapshot"
          rbd -p $pool snap rm $rbd@$snapshot
        fi
        continue
      elif (( weekly > snaptime )); then
        snapday=$(date -d ${snaptime:0:8} +%A)
        if [[ "$snapday" != "Sunday" ]]; then
          echo "$pool/$rbd@$snapshot Deleting snapshot"
          rbd -p $pool snap rm $rbd@$snapshot
        fi
        continue
      elif (( daily > snaptime )); then
        snaphour=${snaptime:8:2}
        if [[ "$snaphour" != "00" ]]; then
          echo "$pool/$rbd@$snapshot Deleting snapshot"
          rbd -p $pool snap rm $rbd@$snapshot
        fi
        continue
      fi
    done &
  done
done