docker: migrate volume data #pg #docker

docker-compose-anonymous.yml

version: '3'
services:
  s1:
    image: postgres:12
    container_name: ${prefix}_compose_anonymous
    ports:
      - $pg_port:5432

docker-compose-bind-mount.yml

version: '3'
services:
  s1:
    image: postgres:12
    container_name: ${prefix}_compose_bind-mount
    ports:
      - $pg_port:5432
    volumes:
      - ./$side-db:/var/lib/postgresql/data

docker-compose-named.yml

version: '3'
services:
  s1:
    image: postgres:12
    container_name: ${prefix}_compose_named
    ports:
      - $pg_port:5432
    volumes:
      - db:/var/lib/postgresql/data
volumes:
  db:

migrate.sh

#!/usr/bin/env bash
set -eu

# USAGE:
# $0 SRC_TYPE:SRC_VOLUME_TYPE DST_TYPE:DST_VOLUME_TYPE
# where SRC_TYPE - standalone, compose
#       SRC_VOLUME_TYPE - anonymous, named, bind-mount
# e.g. $0 standalone:anonymous compose:named

export prefix=vtest
export pg_port=1234
pg_image=postgres:12
src_bind_mount_dir=`realpath ./src-db`
dst_bind_mount_dir=`realpath ./dst-db`

function up {
    local side=$1 type=$2 volume_type=$3 args=()
    [[ $side == src ]] \
        && local bind_mount_dir=$src_bind_mount_dir \
        || local bind_mount_dir=$dst_bind_mount_dir
    if [[ $type == standalone ]]; then
        if [[ $volume_type == named ]]; then
            args+=(--volume "$prefix"_standalone_named:/var/lib/postgresql/data)
        elif [[ $volume_type == bind-mount ]]; then
            args+=(--volume "$bind_mount_dir":/var/lib/postgresql/data)
        fi
        docker run --detach --name "$prefix"_"$type"_"$volume_type" \
            --publish "$pg_port":5432 "${args[@]}" "$pg_image"
    else
        bind_mount_dir=$bind_mount_dir side=$side \
            docker_compose "$volume_type" up -d
    fi
    wait_for_pg
}

function start {
    local side=$1 type=$2 volume_type=$3
    if [[ $type == standalone ]]; then
        docker start "$prefix"_"$type"_"$volume_type"
    else
        side=$side docker_compose "$volume_type" start
    fi
}

function init {
    local side=$1 type=$2 volume_type=$3
    [[ $type == standalone ]] \
        && local cmd=(docker exec "$prefix"_"$type"_"$volume_type") \
        || local cmd=(docker_compose "$volume_type" exec s1)
    side=$side "${cmd[@]}" su - postgres -c 'createdb d1'
    side=$side "${cmd[@]}" su - postgres \
        -c 'psql d1 -c "create table t1 (f1 int)"'
    side=$side "${cmd[@]}" su - postgres \
        -c 'psql d1 -c "insert into t1 (f1) values (1)"'
}

function check {
    local side=$1 type=$2 volume_type=$3
    list_containers; list_volumes
    if [[ $type == standalone ]]; then
        docker exec "$prefix"_"$type"_"$volume_type" su - postgres \
            -c 'psql d1 -c "select * from t1"'
    else
        side=$side docker_compose "$volume_type" exec s1 su - postgres \
            -c 'psql d1 -c "select * from t1"'
    fi
}

function copy {
    local from=${1:-$src_bind_mount_dir} to=${2:-$dst_bind_mount_dir}
    docker run --rm -v "$from:/from" -v "$to:/to" \
        bash -c '
            shopt -s dotglob
            set -x
            ls /to
            rm -r /to/*
            ls /to
            cp -r /from/* /to
        '
}

function stop {
    local side=$1 type=$2 volume_type=$3
    if [[ $type == standalone ]]; then
        docker stop "$prefix"_"$type"_"$volume_type"
    else
        side=$side docker_compose "$volume_type" stop
    fi
}

function rm {
    local side=$1 type=$2 volume_type=$3
    if [[ $type == standalone ]]; then
        docker rm "$prefix"_"$type"_"$volume_type"
    else
        side=$side docker_compose "$volume_type" rm
    fi
}

function down {
    local side=$1 type=$2 volume_type=$3
    if [[ $type == standalone ]]; then
        stop "$side" "$type" "$volume_type"
        docker rm "$prefix"_"$type"_"$volume_type"
    else
        side=$side docker_compose "$volume_type" down
    fi
}

function get_volume_name {
    local c=$1
    command docker inspect "$c" --format '{{(index .Mounts 0).Name}}'
}

function rm_volume {
    local side=$1 v=$2
    [[ $side == src ]] \
        && local bind_mount_dir=$src_bind_mount_dir \
        || local bind_mount_dir=$dst_bind_mount_dir
    if [[ $v ]]; then
        docker volume rm "$v"
    else
        echo
        printf "> sudo chown -R $USER: $bind_mount_dir\n"
        sudo chown -R $USER: "$bind_mount_dir"
        printf "> rm -r $bind_mount_dir\n"
        command rm -r "$bind_mount_dir"
    fi
}

function wait_for_pg {
    sleep 2  # pg gets restarted (you might need to increase this value)
    wait-for-it localhost:"$pg_port"
}

function docker_compose {
    local volume_type=$1
    shift
    echo
    printf "> docker-compose -f docker-compose-$volume_type.yml"
    printf " -p $prefix"
    printf " %s" "$@"
    echo
    docker-compose -f docker-compose-$volume_type.yml -p "$prefix" "$@"
}

function docker {
    echo
    printf "> docker"
    printf " %s" "$@"
    echo
    command docker "$@"
}

function list_containers {
    echo containers:
    command docker container ls -a --format '{{.Names}} {{.ID}} {{.Status}}' \
        | sed -E 's/^/    /'
}

function list_volumes {
    echo volumes:
    command docker volume ls --format '{{.Name}}' \
        | sed -E 's/^/    /'
    if [[ -e ./src-db ]]; then
        echo '    ./src-db'
    fi
    if [[ -e ./dst-db ]]; then
        echo '    ./dst-db'
    fi
}

src=$1
dst=$2
IFS=: read src_type src_volume_type <<< "$src"
IFS=: read dst_type dst_volume_type <<< "$dst"

list_containers; list_volumes
up src "$src_type" "$src_volume_type"
vsrc=`get_volume_name "$prefix"_"$src_type"_"$src_volume_type"`
init src "$src_type" "$src_volume_type"
check src "$src_type" "$src_volume_type"

echo -- start
down  src "$src_type" "$src_volume_type"
up    dst "$dst_type" "$dst_volume_type"
vdst=`get_volume_name "$prefix"_"$dst_type"_"$dst_volume_type"`
stop  dst "$dst_type" "$dst_volume_type"
copy "$vsrc" "$vdst"
rm_volume src "$vsrc"
start dst "$dst_type" "$dst_volume_type"
check dst "$dst_type" "$dst_volume_type"

echo -- cleanup
down dst "$dst_type" "$dst_volume_type"
rm_volume dst "$vdst"
list_containers; list_volumes

output-standalone:anonymous-compose:named.txt

containers:
volumes:

> docker run --detach --name vtest_standalone_anonymous --publish 1234:5432 postgres:12
43098132613b36e6981a2e6a249467302ca0a8f97e607aeff238d4ea49d9f5b8
wait-for-it: waiting 15 seconds for localhost:1234
wait-for-it: localhost:1234 is available after 0 seconds

> docker exec vtest_standalone_anonymous su - postgres -c createdb d1

> docker exec vtest_standalone_anonymous su - postgres -c psql d1 -c "create table t1 (f1 int)"
CREATE TABLE

> docker exec vtest_standalone_anonymous su - postgres -c psql d1 -c "insert into t1 (f1) values (1)"
INSERT 0 1
containers:
    vtest_standalone_anonymous 43098132613b Up 3 seconds
volumes:
    dcbeaa32ff4a935e9b68625eb650d53f1010799dd460c42c3e4bc03e97be7ce5

> docker exec vtest_standalone_anonymous su - postgres -c psql d1 -c "select * from t1"
 f1
----
  1
(1 row)

-- start

> docker stop vtest_standalone_anonymous
vtest_standalone_anonymous

> docker rm vtest_standalone_anonymous
vtest_standalone_anonymous

> docker-compose -f docker-compose-named.yml -p vtest up -d
Creating network "vtest_default" with the default driver
Creating volume "vtest_db" with default driver
Creating vtest_compose_named ... done
wait-for-it: waiting 15 seconds for localhost:1234
wait-for-it: localhost:1234 is available after 0 seconds

> docker-compose -f docker-compose-named.yml -p vtest stop
Stopping vtest_compose_named ... done

> docker run --rm -it -v dcbeaa32ff4a935e9b68625eb650d53f1010799dd460c42c3e4bc03e97be7ce5:/from -v vtest_db:/to bash -c
            shopt -s dotglob
            set -x
            ls /to
            rm -r /to/*
            ls /to
            cp -r /from/* /to

+ ls /to
PG_VERSION            pg_multixact          pg_tblspc
base                  pg_notify             pg_twophase
global                pg_replslot           pg_wal
pg_commit_ts          pg_serial             pg_xact
pg_dynshmem           pg_snapshots          postgresql.auto.conf
pg_hba.conf           pg_stat               postgresql.conf
pg_ident.conf         pg_stat_tmp           postmaster.opts
pg_logical            pg_subtrans
+ rm -r /to/PG_VERSION /to/base /to/global /to/pg_commit_ts /to/pg_dynshmem /to/pg_hba.conf /to/pg_ident.conf /to/pg_logical /to/pg_multixact /to/pg_notify /to/pg_replslot /to/pg_serial /to/pg_snapshots /to/pg_stat /to/pg_stat_tmp /to/pg_subtrans /to/pg_tblspc /to/pg_twophase /to/pg_wal /to/pg_xact /to/postgresql.auto.conf /to/postgresql.conf /to/postmaster.opts
+ ls /to
+ cp -r /from/PG_VERSION /from/base /from/global /from/pg_commit_ts /from/pg_dynshmem /from/pg_hba.conf /from/pg_ident.conf /from/pg_logical /from/pg_multixact /from/pg_notify /from/pg_replslot /from/pg_serial /from/pg_snapshots /from/pg_stat /from/pg_stat_tmp /from/pg_subtrans /from/pg_tblspc /from/pg_twophase /from/pg_wal /from/pg_xact /from/postgresql.auto.conf /from/postgresql.conf /from/postmaster.opts /to

> docker volume rm dcbeaa32ff4a935e9b68625eb650d53f1010799dd460c42c3e4bc03e97be7ce5
dcbeaa32ff4a935e9b68625eb650d53f1010799dd460c42c3e4bc03e97be7ce5

> docker-compose -f docker-compose-named.yml -p vtest start
Starting s1 ... done
containers:
    vtest_compose_named 8fb856c74097 Up Less than a second
volumes:
    vtest_db

> docker-compose -f docker-compose-named.yml -p vtest exec s1 su - postgres -c psql d1 -c "select * from t1"
 f1
----
  1
(1 row)

-- cleanup

> docker-compose -f docker-compose-named.yml -p vtest down
Stopping vtest_compose_named ... done
Removing vtest_compose_named ... done
Removing network vtest_default

> docker volume rm vtest_db
vtest_db
containers:
volumes: