swarm-backuper.sh

#!/bin/bash

# List of all labels:
# docker-backuper.stop=true - stop service (scale=0, scale=UP)
# docker-backuper.exec-pre="pg_dump -U postgres postgres | gzip > /tmp-backup/pg_dump.sql.gz" - command before start
# docker-backuper.volume-list=volume1,volume2,volume3 - which volumes need to backup
# docker-backuper.exec-post="ls -la" - command after backup

IFS=$'\n'

# ---------
# functions
# ---------

# $1 - localServiceName
function waitForServiceComplete() {
  echo "DockerBackuper.waitForServiceComplete: $1"
  localServiceName=$1

  localServiceInfo=$(docker service ps $localServiceName --format json)
  localServiceDesiredState=$(echo $localServiceInfo | jq -r '.DesiredState') # Also = CurrentState
  tryCountSec=0                                                              # 600 sec = 10 min, timeout

  echo "DockerBackuper.waitForServiceComplete.serviceInfoJson:"
  echo "$localServiceInfo"

  while [[ $localServiceDesiredState != "Shutdown" ]] && [[ $tryCountSec -lt 600 ]]; do
    echo "DockerBackuper.waitForServiceComplete.Running, sleep"
    sleep 2
    tryCountSec=$((tryCountSec + 2))
    localServiceInfo=$(docker service ps $localServiceName --format json)
    localServiceDesiredState=$(echo $localServiceInfo | jq -r '.DesiredState')

    echo "DockerBackuper.waitForServiceComplete.serviceInfoJson:"
    echo "$localServiceInfo"
  done
}

# $1 = "docker-backuper.exec-pre" (labelName)
function execLabels() {
  echo "DockerBackuper.execLabels: $1"
  localLabel=$1

  execListRawJson=$(docker service ls --filter label="$localLabel" --format json)
  echo "DockerBackuper.Exec.serviceList:"
  echo "$execListRawJson"

  for serviceEl in $execListRawJson; do
    echo "DockerBackuper.Exec.serviceEl:"
    echo "$serviceEl"
    serviceId=$(echo $serviceEl | jq -r '.ID')
    serviceName=$(echo $serviceEl | jq -r '.Name') # vikunja-pg_master; vikunja-pg=STACK, master=SERVICE

    serviceInspectJson=$(docker inspect $serviceId --format json)
    execLabel=$(echo $serviceInspectJson | jq -r ".[0].Spec.Labels.\"$localLabel\"")
    echo "DockerBackuper.Exec.execLabel: $execLabel"

    taskListRawJson=$(docker service ps $serviceName --filter 'desired-state=running' --format json)
    echo "DockerBackuper.Exec.taskListRawJson:"
    echo "$taskListRawJson"

    for taskEl in $taskListRawJson; do
      echo "DockerBackuper.Exec.taskEl:"
      echo "$taskEl"

      taskId=$(echo $taskEl | jq -r '.ID')     # 380xcsrylpmc0wvl5ntd3xfa5
      taskName=$(echo $taskEl | jq -r '.Name') # nginx_master.2
      taskNode=$(echo $taskEl | jq -r '.Node') # internal-manager-1 (hostname)

      taskInspectJson=$(docker inspect $taskId --format json)
      containerId=$(echo $taskInspectJson | jq -r '.[0].Status.ContainerStatus.ContainerID')

      echo "DockerBackuper.Exec.ready-to-go:"
      echo "serviceId=$serviceId, serviceName=$serviceName, containerId=$containerId, taskId=$taskId, taskName=$taskName, taskNode=$taskNode"

      execCommand="docker exec $containerId /bin/sh -c '$execLabel'"
      echo "ExecCommand: $execCommand"

      execServiceName="docker_backuper_exec"

      docker service create \
        --detach \
        --name $execServiceName \
        --mode replicated \
        --replicas 1 \
        --constraint node.hostname==$taskNode \
        --restart-condition none \
        --mount type=bind,source=/var/run/docker.sock,destination=/var/run/docker.sock,readonly \
        docker:25.0.5-cli-alpine3.20 sh -c "$execCommand"

      # While loop here
      waitForServiceComplete $execServiceName

      docker service logs $execServiceName
      docker service remove $execServiceName
    done
  done
}

# ---------
# Get all volumes, from all nodes
# Transform them to Map
# ---------
echo "DockerBackuper.VolumeCollector.INIT"

nodeListRawJson=$(docker node ls --format json)
echo "DockerBackuper.VolumeCollector.NodeList"
echo "$nodeListRawJson"
declare -A volumeNodeListMap

for nodeEl in $nodeListRawJson; do
  echo "DockerBackuper.VolumeCollector.nodeEl:"
  echo "$nodeEl"

  nodeId=$(echo $nodeEl | jq -r '.ID')
  nodeName=$(echo $nodeEl | jq -r '.Hostname') # internal-worker-1
  nodeStatus=$(echo $nodeEl | jq -r '.Status') # Ready

  if [[ $nodeStatus == "Ready" ]]; then
    serviceName="volume_collector"

    docker service create \
      --detach \
      --name $serviceName \
      --replicas 1 \
      --constraint node.id==$nodeId \
      --restart-condition none \
      --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \
      docker:25.0.5-cli-alpine3.20 \
      sh -c 'docker volume ls -f driver=local --format json'

    # Wait for complete
    waitForServiceComplete $serviceName

    # Put all volumes from all nodes into MAP<volumeName, nodeList>
    serviceLogListJson=$(docker service logs $serviceName --raw)
    echo "DockerBackuper.VolumeCollector.Logs:"
    echo "$serviceLogListJson"

    # Remove the service
    docker service remove $serviceName

    for logEl in $serviceLogListJson; do
      echo "DockerBackuper.VolumeCollector.logEl:"
      echo "$logEl"

      volumeName=$(echo $logEl | jq -r '.Name') # vikunja-pg-backup-data

      if [ -v volumeNodeListMap["$volumeName"] ]; then
        echo "$volumeName exist"
        nodeList="${volumeNodeListMap[$volumeName]}"
        nodeList+=",$nodeName"
        volumeNodeListMap[$volumeName]=$nodeList
      else
        echo "$volumeName NOT exist"
        nodeList="$nodeName"
        volumeNodeListMap[$volumeName]=$nodeList
      fi
    done
  fi
done
echo "DockerBackuper.VolumeCollector.OK"

# ---------
# Get all volumes, which need to backup
# ---------
# docker-backuper.volume-list=gitlab-backup-data
# docker-backuper.volume-list=kek-data-1,lol-data-2,www-data-3
echo "DockerBackuper.VolumeLabels.INIT"

seviceVolumeLabelListRawJson=$(docker service ls --filter label="docker-backuper.volume-list" --format json)
echo "DockerBackuper.VolumeLabels.seviceVolumeLabelListRawJson"
echo "$seviceVolumeLabelListRawJson"
declare -A nodeVolumeListMap

for serviceEl in $seviceVolumeLabelListRawJson; do
  echo "DockerBackuper.VolumeLabels.serviceEl"
  echo "$serviceEl"

  serviceId=$(echo $serviceEl | jq -r '.ID')
  serviceName=$(echo $serviceEl | jq -r '.Name') # vikunja-pg_master; vikunja-pg=STACK, master=SERVICE

  serviceInspectJson=$(docker inspect $serviceId --format json)
  volumeListLabel=$(echo $serviceInspectJson | jq -r '.[0].Spec.Labels."docker-backuper.volume-list"')

  IFS=$',' # Change, volumeListLabel=volume1,volume2,volume3
  for volumeName in $volumeListLabel; do
    if [ -v volumeNodeListMap["$volumeName"] ]; then
      echo "DockerBackuper.VolumeLabels.Volume: $volumeName EXIST"
      nodeList="${volumeNodeListMap[$volumeName]}"

      for nodeName in $nodeList; do
        if [ -v nodeVolumeListMap["$nodeName"] ]; then
          echo "DockerBackuper.VolumeLabels.Node: $nodeName EXIST"
          volumeList="${nodeVolumeListMap[$nodeName]}"
          volumeList+=",$volumeName"
          nodeVolumeListMap[$nodeName]=$volumeList
        else
          echo "DockerBackuper.VolumeLabels.Node: $nodeName NOT EXIST"
          volumeList="$volumeName"
          nodeVolumeListMap[$nodeName]=$volumeList
        fi
      done
    fi
  done
  IFS=$'\n' # Return IFS
done
echo "DockerBackuper.VolumeLabels.OK"

# ---------
# Get all services with label docker-backuper.exec-pre
# ---------
echo "DockerBackuper.ExecPre.INIT"
label="docker-backuper.exec-pre"
execLabels $label
echo "DockerBackuper.ExecPre.OK"

# ---------
# Stop all services with label docker-backuper.stop=true, scaleDown=0
# Work only with Mode=Replicated
# ---------
echo "DockerBackuper.StopLabel.INIT"
stopListRawJson=$(docker service ls --filter mode=replicated --filter label="docker-backuper.stop=true" --format json)
echo "DockerBackuper.StopLabel.stopListRawJson:"
echo "$stopListRawJson"

for serviceEl in $stopListRawJson; do
  echo "DockerBackuper.StopLabel.serviceEl:"
  echo "$serviceEl"

  serviceName=$(echo $serviceEl | jq -r '.Name') # vikunja-pg_master; vikunja-pg=STACK, master=SERVICE

  docker service scale $serviceName=0
done
echo "DockerBackuper.StopLabel.OK"

# ---------
# for loop nodeVolumeListMap and USE offen-backup
# ---------
echo "DockerBackuper.OffenBackup.INIT"
for nodeName in "${!nodeVolumeListMap[@]}"; do
  echo "DockerBackuper.OffenBackup.for-map:"
  echo "nodeName: $nodeName"
  echo "volumeList: ${nodeVolumeListMap[$nodeName]}"

  serviceName="offen_backup"

  backupCommand="docker service create --detach --name $serviceName --replicas 1 --constraint node.hostname==$nodeName --restart-condition none --entrypoint \"/bin/sh\""
  backupCommand+=" -e BACKUP_CRON_EXPRESSION=\"0 0 5 31 2 ?\" -e BACKUP_RETENTION_DAYS=5 -e BACKUP_COMPRESSION=gz -e BACKUP_FILENAME=backup-$nodeName-%Y-%m-%dT%H-%M-%S.tar.gz"
  backupCommand+=" -e AWS_ENDPOINT=... -e AWS_S3_BUCKET_NAME=... -e AWS_ACCESS_KEY_ID=... -e AWS_SECRET_ACCESS_KEY=..."

  IFS=$',' # Change, volumeList=volume1,volume2,volume3
  volumeList="${nodeVolumeListMap[$nodeName]}"
  for volumeName in $volumeList; do
    backupCommand+=" --mount type=volume,source=$volumeName,target=/backup/$volumeName"
  done
  IFS=$'\n'

  backupCommand+=" offen/docker-volume-backup:v2.39.1 -c 'backup && exit'"

  echo "DockerBackuper.OffenBackup.backupCommand:"
  echo "$backupCommand"

  # Execute command from variable
  eval "$backupCommand"

  # While loop here
  waitForServiceComplete $serviceName

  docker service logs $serviceName
  docker service remove $serviceName
done
echo "DockerBackuper.OffenBackup.OK"

# ---------
# Restore services replicas count
# ---------
echo "DockerBackuper.StopLabel.Restore.INIT"
for serviceEl in $stopListRawJson; do
  echo "DockerBackuper.StopLabel.Restore.serviceEl:"
  echo "$serviceEl"

  serviceName=$(echo $serviceEl | jq -r '.Name')             # vikunja-pg_master; vikunja-pg=STACK, master=SERVICE
  replicasCountString=$(echo $serviceEl | jq -r '.Replicas') # 1/1, 5/10
  replicasCount="${replicasCountString#*\/}"                 # 1/1 -> 1

  docker service scale $serviceName=$replicasCount
done
echo "DockerBackuper.StopLabel.Restore.OK"

# ---------
# Execute post script
# ---------
echo "DockerBackuper.ExecPost.INIT"
label="docker-backuper.exec-post"
execLabels $label
echo "DockerBackuper.ExecPost.OK"