Incrementally gzip a file (such as a log). Each call will append only new bytes from the input to the output.

gzip-cont.sh

#!/bin/bash

# Usage: gzip-cont {in_file} {out_file} {gzip_args...}

set -e -o pipefail

in="$1"
out="$2"
shift 2

if [ -f "${out}.gzip-cont" ]; then
  offset=$(cat "${out}.gzip-cont")
else
  offset=0
fi

size=$(stat --printf="%s" "$in")
prev_size=$(stat --printf="%s" "$out" || echo 0) 

cancel() {
  truncate -s "$prev_size" "$out"
  exit 1
}
trap cancel INT
if ! dd if="$in" skip="$offset" count="$(( size - offset ))" iflag=skip_bytes,count_bytes | pv | gzip "$@" >> "$out"; then 
  cancel
fi

echo "$size" > "${out}.gzip-cont"