Ansible playbook to move persistent journald logs to loopback mounted Btrfs filesystem with compression. Makes sure that journal files are kept decently compressed

compressed_journald_logs.yaml

    # FIXME: override /usr/lib/tmpfiles.d/journal-nocow.conf to avoid applying noCOW flag?
    # systemd-tmpfiles --create?
    #
    # FIXME: systemd 250 reenables COW for archived journal. so we can skip parts of configuration.

- name: Store journald logs on compressed volume
  hosts: all
  gather_facts: yes
  become: yes

  vars_prompt:
    - name: volume_size_mib
      prompt: Choose size for compressed btrfs storage (MiB)
      private: False

  vars:
    volume_location: /var/log/journal.btrfs
    compressed_journal_location: /var/log/journal

    # I usually get compression ratios around 0.12 - 0.15 but let's err on safe side
    expected_compression_ratio: 0.2

    SystemMaxFileSize_mib: "{{ (volume_size_mib | int / 16) | int }}"

    bind_of_original_log: /var/log_original
    recompression_timestamp_file: /var/log/journal/recompression_timestamp

  tasks:
    - name: Check that /var/log/journal is not mounted yet
      ansible.builtin.assert:
        that:
          - "{{ ansible_facts.mounts | selectattr('mount', 'equalto', '/var/log/journal') | length == 0}}"
        fail_msg: '/var/log/journal already mounted. Migration of mounted /var/log/journal is not supported'


    - name: Get size of current journal data
      command: du -sk /var/log/journal
      changed_when: False
      register: var_log_journal_du

    - set_fact:
        original_journal_dir_size_kib: "{{ var_log_journal_du.stdout.split()[0] | int }}"
    - set_fact:
        minimal_compressed_volume_size_mib: "{{ ((original_journal_dir_size_kib | int) * expected_compression_ratio / 1024 ) | int }}"

    - name: Old journal data fits new partition
      ansible.builtin.assert:
        that:
          - "{{ (minimal_compressed_volume_size_mib | int) <= ( volume_size_mib | int) }}"
        fail_msg: >-
          Your original journal data probably will not fit in compressed journal size that you have requested.
          You should allocate at least {{ minimal_compressed_volume_size_mib }} MiB for compressed volume.

    - name: Check that distro family is supported
      ansible.builtin.assert:
        that:
          - "{{ ansible_pkg_mgr in ['apt', 'pacman'] }}"
        fail_msg: "Only apt and pacman package managers are supporter. Cannot check and install dependencies."

    - name: Install dependencies (deb)
      when: ansible_pkg_mgr == 'apt'
      apt:
        update_cache: yes
        name:
          - btrfs-compsize
          - btrfs-progs
          - e2fsprogs #chattr
          - rsync

    - name: Install dependencies (arch)
      when: ansible_pkg_mgr == 'pacman'
      community.general.pacman:
        update_cache: yes
        name:
          - btrfs-progs
          - compsize
          - e2fsprogs #chattr
          - rsync

    # community.general.filesize is too new and uses dd which might be slow
    - name: Create block device for storage
      ansible.builtin.command:
        # FIXME: umask! Important for security
        cmd: fallocate --length {{ volume_size_mib }}MiB {{ volume_location }}
        creates: "{{ volume_location }}"
          # TODO: mark as nocow

    - name: Proper permission on storage block device
      ansible.builtin.file:
        path: "{{ volume_location }}"
        owner: root
        group: root
        mode: '0600'

    - name: Setup filesystem
      community.general.filesystem:
        dev: "{{ volume_location }}"
        fstype: btrfs
        # I actually do not know when resize support for btrfs was added to this module, but in Ansible 2.10.8 it is not ailable
        resizefs: "{{ ansible_version.full is version('2.10.8', '>') }}"
        opts: "{{ ( (volume_size_mib | int) < 4096 ) | ternary('--mixed', omit) }}"

    - name: Journald config
      tags: [journald_config]
      community.general.ini_file:
        path: /etc/systemd/journald.conf
        section: Journal
        backup: yes
        create: no
        option: "{{ item.key   }}"
        value:  "{{ item.value }}"
      loop: "{{ journald_params | dict2items }}"
      vars:
        journald_params:
          SystemMaxUse: "{{ ((volume_size_mib | int) / expected_compression_ratio) | int }}M"  #Probably should give more space?
          # Somewhere I see journal rotate before reaching SystemMaxFileSize. Dunno why.
          SystemMaxFileSize: "{{ SystemMaxFileSize_mib }}M"
          SystemKeepFree: "{{ SystemMaxFileSize_mib | int + 16 }}M"
          SystemMaxFiles: "{{ volume_size_mib | int * 4}}" #default is 100. Let's store everything that fits
          Compress: 'no'

    - name: Rotate journald logs to save existing logs in archive
      ansible.builtin.command:
        cmd: journalctl --rotate

    - name: Mount compressed journal volume
      ansible.posix.mount:
        path: "{{ compressed_journal_location }}"
        src: "{{ volume_location }}"
        state: mounted
        fstype: btrfs
        opts: 'rw,compress=zstd:8'
        backup: yes

    # All logged data from this point till systemd-journald restart will be lost

    - name: Restore access to underlying journald logs
      ansible.posix.mount: &bind_mount
        path: "{{ bind_of_original_log }}"
        src: /var/log/
        opts: bind
        state: mounted
        fstype: none

    # Without existing directory for machine logs systemd-journald creates it
    # but system.journal file that it creates lacks proper permissions.
    # Therefore has to create directories before resarting logging to new volume
    # `systemd-tmpfiles --create` did not work here.
    - name: Restore original directory layout
      ansible.posix.synchronize:
        src: "{{ bind_of_original_log }}/journal/"
        dest: "{{ compressed_journal_location }}/"
        rsync_opts:
          - '--acls'
          - '--xattrs'
          # Include only dirs
          - '--include=*/'
          - '--exclude=*'
      delegate_to: "{{ inventory_hostname }}"

    - name: Restart systemd-journald to start writing to new location
      tags: [journald_config]
      ansible.builtin.systemd:
        name: systemd-journald
        state: restarted

          # FIXME: this should be a last action?
    - name: Copy original journal files archive (this takes a while)
      ansible.posix.synchronize:
        src: "{{ bind_of_original_log }}/journal/"
        dest: "{{ compressed_journal_location }}/"
        # By default synchronize task calls uses rsync compression but this does not make sence on local system
        # Disabling it decreased transfer time 5x times on test system
        compress: no
        rsync_opts:
          - '--acls'
          - '--xattrs'
          - '--hard-links'
          # Exclude current system and user files by excluding everything and including dirs and archived files
          - '--include=*/'
          - '--include=*@*.journal'
          - '--include=*@*.journal~'  #Improperly closed files?
          - '--exclude=*'
      delegate_to: "{{ inventory_hostname }}"


    - name: Suggest running cleanup actions
      tags: always
      debug:
        msg: >-
          Rerun this playbook with `--tags cleanup` to cleanup old journald data and temporary bind mount
          when you confirm that journal transfer worked properly.
      when: "'cleanup' not in ansible_run_tags"

    - name: Remove old data and temporary bind mount
      tags: [never, cleanup]    # Skip unless explicitly requested
      block:
        - name: Remove uncompressed journald data from underlying storage
          shell:
            cmd: "rm -r '{{ bind_of_original_log }}/journal/'*"
            warn: false  # Can't use file module because it would remove journal dir itself
        - name:
          ansible.posix.mount:
            <<: *bind_mount
            state: absent


    - name: Setup recompression of rotated files
      tags: maintenance
      block:
        - name: Last recompression timestamp
          ansible.builtin.file:
            path: "{{ recompression_timestamp_file }}"
            state: touch
            access_time:       preserve
            modification_time: preserve

        - name: Override nocow policy for journald dir
          copy:
            dest: /etc/tmpfiles.d/journal-nocow.conf
            content: |
              # This is an empty file to override default systemd policy of setting No_COW attribute on journald directories.
              # It is rumored that No_COW gives better performance on Btrfs
              # but it actually prevents file compression and defragmentation
              # which leads to wasted space and reduced performance

        - name: Path file to monitor changes
          ansible.builtin.copy:
            dest: /etc/systemd/system/journal_rotated.path
            backup: yes
          # validate: 'systemd-analyze verify %s'
            content: |
              [Path]
              # Reacts to new files
              PathChanged=/var/log/journal/{{ ansible_facts.machine_id }}
              Unit=journal_compress.service

              [Install]
              # FIXME: this watcher has to be an essential dependency of systemd-journald
              WantedBy=multi-user.target

        - name: Service to recompress journal files
          ansible.builtin.copy:
            dest: /etc/systemd/system/journal_compress.service
            backup: yes
            # Validation is tricky. systemd requires that service file ends with .service Otherwise verify fails
            # But temporary file is created with another extension. Same issue:
            # https://groups.google.com/g/ansible-project/c/AoUsjROS4b0?pli=1
            # Approach from ansible docs too complicated
            # https://docs.ansible.com/ansible/devel/reference_appendices/faq.html#the-validate-option-is-not-enough-for-my-needs-what-do-i-do
            # validate: 'systemd-analyze verify %s'
            content: |
              [Unit]
              #FIXME: description (all units)
              Description=Keep journal files compressed, see https://juick.com/Self-Perfection/2873646

              [Service]
              Type=oneshot
              User=root
              WorkingDirectory=/var/log/journal

              ExecStart=/usr/bin/env touch {{ recompression_timestamp_file }}-new


              # Prior to defragment make sure there are no files/dirs marked with no COW attribute
              # It keeps appearing for unknown reason (probably on boot due to tmpfiles)
              ExecStart=/usr/bin/env find . -type d -exec chattr -V -C {} +

              # OK just defragment everything
              # Pro: don't complain on defragment attempt of deleted files
              # Contra: caches whole journal contents in page cache

              # Does not reduce space usage since ~ 2022-01-19 ((( Probably should be dropped
              # FIXME: reads whole journal content. How to skip already defragmented files?
              # Use timestamp file marker! Then move newer one on top
              # `btrfs filesystem defragment -czstd $FILE` is not efficient as it does not reduce space wasted on prealloc
              ExecStart=/usr/bin/env find -D exec . -type f ( -name '*@*.journal~' -o -name '*@*.journal' ) -newer '{{ recompression_timestamp_file }}' -print -exec sh -xc 'cp --reflink=never -av "$${0}" "$${0}_repack" && mv -v "$${0}_repack" "$${0}"' {} \;
              ExecStart=/usr/bin/env mv -v {{ recompression_timestamp_file }}-new {{ recompression_timestamp_file }}
              # FIXME: add fallocate --dig-holes ??? might improve compression as journald allocates by 8MB chunks but not always fills them
              # Alternative: cp --sparse=always , --sparse=auto, --sparse=never, sync, retain smallest files
              # cp sparse=always might produce bigger and smaller file. dig-holes seems to always reduce file size
              # but probably leads to fragmentation?

        - name: Enable path
          ansible.builtin.systemd:
            name: journal_rotated.path
            daemon_reload: yes
            state: started
            enabled: yes