Automatically persist memory state for a Docker container using CRIU

run_with_checkpoints.sh

#!/bin/bash
# A bash script that will periodically checkpoint a container, maintaining a maximum of five checkpoints.
# Once five checkpoints have been created, it will periodically replace the oldest checkpoint with a new one.
# Arguments:
# - run_args (a single string containing the arguments to pass to 'docker run')
# - container_name (name of the existing container to run with automatic persistence or name to give to new container)
# - container_image (image to run in the container)
# - container_args (arguments to pass to the service running in the container)

run_args=$1
container_name=$2
container_image=$3
shift 3
container_args=$*

# Where to store checkpoints
datadir=/var/${container_name}_data
maxcheckpoints=5

# Regex to extract checkpoint number from checkpoint name
checkpoint_re="[^0-9]*([0-9]*)"

# Create the data directory if necessary
ls ${datadir} &>/dev/null || (sudo mkdir ${datadir}; sudo chown jrj:jrj ${datadir})

# Create a rolling set of backup checkpoints for the running container
backups() {
    while true; do
        sleep 5

        # Check status of container and exit with error if no container exists
        status=$(docker container inspect -f '{{.State.Status}}' ${container_name} 2>/dev/null) || exit 1

        # If container is stopped, exit without error
        [[ "${status}" != "running" ]] && exit 0

        # Find the current number of checkpoints
        cpcount=$(ls ${datadir} 2>/dev/null | wc -w)

        # If we've reached max, remove oldest checkpoint and
        # set next checkpoint number to number of oldest
        if [[ ${cpcount} -ge ${maxcheckpoints} ]]; then
            oldest=$(ls -rt ${datadir} 2>/dev/null | head -1)
            i=$([[ ${oldest} =~ ${checkpoint_re} ]] && echo ${BASH_REMATCH[1]})
            docker checkpoint rm --checkpoint-dir=${datadir} ${container_name} ${oldest}
        else
            # otherwise, set next checkpoint number to next open number
            i=$((${cpcount} + 1))
        fi

        docker checkpoint create --checkpoint-dir=${datadir} \
               --leave-running=true ${container_name} checkpoint${i}
    done
}

# Create a container if it does not yet exist
docker container inspect ${container_name} &>/dev/null \
    || docker container create --name ${container_name} ${run_args} ${container_image}

# Find the most recent checkpoint
latest=$(ls -t ${datadir} 2>/dev/null | head -1)

if [[ "${latest}" != "" ]]; then
    # If a checkpoint exists, use it to start the container
    docker container start --checkpoint=${latest} --checkpoint-dir=${datadir} ${container_name}
else
    # If no checkpoint exists, start the container without one
    docker container start ${container_name}
fi

# Start running backup checkpoints
backups