backup with ansible + restic + rclone (local backup included); server setup for docker with optional namespace remapping

backup.sh.j2

#!/usr/bin/env bash

echo "-- Starting backup script"

set -e
set -o pipefail
umask o=rwx,go=

# TODO:
# - make script secured, remove credentials from env and process tree

# -- remove all created working files on script exit
working_files=()

finalize() {
  # Remove files from host
  for file in "${working_files[@]}"; do
    rm -rf "${file}"
  done
}

trap finalize EXIT ERR

init_restic_repo() {
  if ! RESTIC_PASSWORD={{ restic_password | mandatory }} restic --repo="{{ restic_repo | mandatory }}" cat config > /dev/null; then
    # then init new one
    RESTIC_PASSWORD={{ restic_password | mandatory }} restic --repo="{{ restic_repo | mandatory }}" init || {
      echo "Failed initializing restic repository"
      return 1
    }
  fi
}

notify_to_telegram() {
  {% if telegram_bot_token is defined and telegram_chat_id is defined %}

  text="<b>{{ restic_host }} $1</b>\n$2"
  message_json="{
    \"chat_id\": {{ telegram_chat_id }},
    \"text\": \"$text\",
    \"parse_mode\": \"html\",
    \"disable_notification\": true
  }"

  curl --silent --show-error --fail -X POST \
    -H 'Content-Type: application/json' \
    -d "$message_json" https://api.telegram.org/bot{{ telegram_bot_token }}/sendMessage > /dev/null || \
      { echo "sending to telegram failed" && return 0; }

  {% endif %}

  return 0
}

{# all backup targets should be collected in this var #}
{% set final_backup_targets = backup_targets | default([]) %}

do_backup() {
  # -- mysql backup (inside container)
  {% if mysql_database is defined %}

  mysql_id=$(docker ps --filter "ancestor=mysql" -q) || return 1
  [ -z "$mysql_id" ] && { echo "mysql not running inside container" && return 1; }

  docker exec $mysql_id sh -e -c \
    'mysqldump --user={{ mysql_user }} --password={{ mysql_password }} {{ mysql_database }}' > \
    /home/{{ app_user }}/{{ mysql_database }}_mysql_dump.sql || return 1

  working_files+=({{ mysql_database }}_mysql_dump.sql)
  {{ final_backup_targets.append(mysql_database + '_mysql_dump.sql') }}

  {% endif %}

  # -- pg backup (inside host)
  {% if pg_database is defined %}

  pg_dump --clean {{ pg_database }} > /home/{{ app_user }}/{{ pg_database }}_pg_dump.sql || return 1

  working_files+=({{ pg_database }}_pg_dump.sql)
  {{ final_backup_targets.append(pg_database + '_pg_dump.sql') }}

  {% endif %}


  # -- do backup
  backup_log=$(RESTIC_PASSWORD={{ restic_password | mandatory }} \
    restic -v --host={{ restic_host | mandatory }} --repo={{ restic_repo | mandatory }} \
    backup {{ final_backup_targets | join(' ') }}) || {
      echo "backup failed"
      return 1
    }

  printf "$backup_log\n"
  filtered_log=$(printf "$backup_log" | sed -n -E -e "/using parent|start scan/,\$p") || return 1
  notify_to_telegram "backup completed success" "$filtered_log"
}

run() {
  init_restic_repo || return 1
  do_backup || return 1
}

run || {
  notify_to_telegram "backup failed" "inspect errors in backup.log"
}

docker-daemon.json.j2

{
  "userns-remap":"{{ app_user }}"
}

local_backup.playbook.yml

---
- name: Daily local backup
  hosts: 127.0.0.1
  connection: local
  tags: backup
  vars_files:
    - credentials.yml
  vars:
    app_user: vorobey
    restic_host: macbook16.local
    restic_repo: rclone:yandex-s3:pechorin.local-backup-test
    restic_password: "{{ pechorindev_restic_password }}"
    backup_targets:
      - /Users/vorobey/ansible.private-infra
  tasks:
    - name: create daily backup script
      template:
        src: 'files/backup.sh.j2'
        dest: "/Users/{{ app_user }}/backup.daily.sh"
        owner: "{{ app_user }}"
        mode: '0700'

    - name: setup daily cron backup
      ansible.builtin.cron:
        name: "every day backup"
        user: "{{ app_user }}"
        special_time: daily
        job: "/Users/{{ app_user }}/backup.daily.sh &>> backup.daily.log"

restore.sh.j2

#!/usr/bin/env bash

echo "-- Starting restore script"

set -e
set -o pipefail
umask o=rwx,go=

{% set restore_folder = '/home/' + app_user + '/restore_' + restic_host  %}
rm -vrf {{ restore_folder }}
mkdir -v {{ restore_folder }}

RESTIC_PASSWORD={{ restic_password | mandatory }} \
  restic -v --host={{ restic_host | mandatory }} \
            --repo={{ restic_repo | mandatory }} restore latest --target={{ restore_folder }}

roles_webserver_tasks_main.yml

---
# tasks file for webserver
- name: Install webserver packages
  become: true
  apt:
    pkg: "{{ item }}"
    update_cache: true
    state: present
  with_items:
    - neovim
    - rclone
    - restic
    - docker.io
    - openssl

- name: Add User
  ansible.builtin.user:
    name: "{{ app_user }}"
    shell: /bin/bash
    groups: docker
    append: yes

- name: Install ssh key
  ansible.builtin.authorized_key:
    user: "{{ app_user }}"
    state: present
    key: https://github.com/pechorin.keys

- name: Create rclone config directory
  ansible.builtin.file:
    path: /home/vorobey/.config/rclone/
    owner: "{{ app_user }}"
    group: "{{ app_user }}"
    mode: '0700'
    state: directory
    recurse: true

- name: Copy rclone configuration
  ansible.builtin.copy:
    src: ~/.config/rclone/rclone.conf
    dest: "/home/{{ app_user }}/.config/rclone/rclone.conf"
    owner: "{{ app_user }}"
    group: "{{ app_user }}"
    mode: '0700'

- name: Do docker namespace remapping
  tags: docker
  when: do_docker_ns_remap
  vars:
    rearrange_required_pattern: "{{ app_user }}:100000:65536\n{{ app_user }}:1000:1"
  block:
    - name: Add 1:1000 mapping to host
      ansible.builtin.command: "usermod -v 1000-1000 -w 1000-1000 {{ app_user }}"

    - name: Get subuid file
      ansible.builtin.command: "cat /etc/subuid"
      register: subuid

    - name: Rearrange lines (very naive implementation)
      block:
        - name: Intro
          ansible.builtin.debug:
            msg: "Lets do the rearrange"

        # default uids mapping for ubuntu will be
        #
        #   username:100000:65536
        #
        # we will map docker root to our host user via appending line
        #
        #   username:1000:1
        #
        # after what we will swap lines order, so 1000:1 will be first
        - name: Swap mapping line order
          ansible.builtin.shell:
            cmd: |
              tac /etc/subuid > ./temp_subuid
              cat ./temp_subuid > /etc/subuid

              tac /etc/subgid > ./temp_subgid
              cat ./temp_subgid > /etc/subgid

              rm -f ./temp_subuid ./temp_subgid

      when: subuid.stdout is match(rearrange_required_pattern)

    - name: Install daemon.json
      ansible.builtin.template:
        src: docker-daemon.json.j2
        dest: /etc/docker/daemon.json
        backup: true

- name: Start docker service
  service:
    name: docker
    state: started

webservers.playbook.yml

---
- name: pechorin.dev host
  hosts: pechorin.dev
  remote_user: root
  tags: setup
  roles:
    - role: webserver
      vars:
        app_user: vorobey
        
 - name: pechorin.dev backup
  hosts: pechorin.dev
  tags: backup
  remote_user: root
  vars_files:
    - credentials.yml
  vars:
    app_user: vorobey
    restic_host: pechorin.dev
    restic_repo: rclone:yandex-s3:pechorin.dev-db-backup-test
    restic_password: "{{ pechorindev_restic_password }}"
    mysql_user: root
    mysql_database: wordpress
    mysql_password: "{{ pechorindev_mysql_password }}"
    backup_targets:
      - /var/lib/docker/1000.1000/volumes/pechorindev_wp_data/_data
  tasks:
    - name: upload backup script
      template:
        src: 'files/backup.sh.j2'
        dest: "/home/{{ app_user }}/backup.sh"
        owner: "{{ app_user }}"
        group: "{{ app_user }}"
        mode: '0700'
    - name: upload restore script
      template:
        src: 'files/restore.sh.j2'
        dest: "/home/{{ app_user }}/restore.sh"
        owner: "{{ app_user }}"
        group: "{{ app_user }}"
        mode: '0700'
    - name: set cron backup
      ansible.builtin.cron:
        name: "every day backup"
        user: "{{ app_user }}"
        special_time: daily
        job: "/home/{{ app_user }}/backup.sh &>> backup.log"