事前に zstd にして zip ファイルをコンテナとして扱うやつ

README.md

zipstd

Zstandard compression in zip.

Install

$ make test
$ INSTALL_DIR=[zipstd install directory] make install

Usage

Compression

Usage:
  zipstd [OPTIONS] files

Options:
  -h, --help
  -#: compression level, 1-19 (default: 3)
  -P, --processes NUMBER: Maximum number of processes (default: 1)
  -o FILENAME: output file (default: out.zip)
  -v: verbose mode
  -W: overwrite if output file already exists
  -x PATTERN:  no compress file pattern

decompression

Usage:
  unzipstd [OPTIONS] files

Options:
  -h, --help
  -P, --processes ARG: Maximum number of processes (default: 1)
  -o ARG: output directory (default: .)
  -v: verbose mode

License

MIT

Makefile

test:
	which /bin/bash || { echo "please install bash to /bin/bash"; exit 1 ; }
	which base64    || { echo "please install base64";            exit 1 ; }
	which find      || { echo "please install find";              exit 1 ; }
	which sed       || { echo "please install sed";               exit 1 ; }
	which unzip     || { echo "please install unzip";             exit 1 ; }
	which xargs     || { echo "please install xargs";             exit 1 ; }
	which zip       || { echo "please install zip";               exit 1 ; }
	which zstd      || { echo "please install zstd";              exit 1 ; }

install: test
	[ -n "${INSTALL_DIR}" ] || { echo "Usage:\n\tINSTALL_DIR=Directory make install"; exit 1 ; }
	[ -d "${INSTALL_DIR}" ] || { echo "${INSTALL_DIR} not exists";                    exit 1 ; }
	install zipstd.sh "${INSTALL_DIR}/zipstd"
	install unzipstd.sh "${INSTALL_DIR}/unzipstd"

unzipstd.sh

#!/bin/bash

declare -i argc=0
declare -a argv=()

progname=$(basename $0)
outdir="."
procs=1
varbose=""

usage() {
  echo "Usage:"
  echo "  ${progname} [OPTIONS] files"
  echo
  echo "Options:"
  echo "  -h, --help"
  echo "  -P, --processes ARG: Maximum number of processes (default: ${procs})"
  echo "  -o ARG: output directory (default: ${outdir})"
  echo "  -v: verbose mode"
  echo
  exit 1
}

: "parse options" && {
  while [ -n "$1" ]
  do
    OPT=$1
    case $OPT in
      '-h' | '--help' )
        usage
        ;;
      '-P' | '--processes' )
        if [[ -z $2 ]] || [[ $2 =~ ^-+ ]] ; then
          echo "${progname}: option requires an argument -- $1" 1>&2
          exit 1
        fi
        procs=$2
        shift 2
        ;;
      '-o' )
        if [[ -z $2 ]] || [[ $2 =~ ^-+ ]] ; then
          echo "${progname}: option requires an argument -- $1" 1>&2
          exit 1
        fi
        outdir=$2
        shift 2
        ;;
      '-v' )
        verbose="yes"
        shift
        ;;
      -*)
        echo "${progname}: illegal option -- '$(echo $1 | sed 's/^-*//')'" 1>&2
        exit 1
        ;;
      *)
        ((++argc))
        argv+=( "$1" )
        shift
        ;;
    esac
  done

  if [ -z $argv ]; then
    echo "${progname}: too few arguments" 1>&2
    usage
  fi
}

: "main" && {
  echo "${progname}: start decompression"
  echo "${progname}: Maximum number of Processes: ${procs}"

  decompress() {
    file=$1
    echo "[File: ${file}]"
    unzip ${file} -d ${outdir}
    find ${outdir} -type f -name "*.zst" -print0 | xargs ${verbose:+-t} -0 -n 1 -P ${procs} zstd -d --rm
    echo
  }

  for file in ${argv[@]}
  do
    if [ -f $file ]; then
      decompress $file
    fi
    if [ -d $file ]; then
      echo "${progname}: skip. ${file} is directory"
    fi
  done
}

zipstd.sh

#!/bin/bash

declare -i argc=0
declare -a argv=()

progname=$(basename $0)
outfile="out.zip"
srcdir=$1
procs=1
overwrite=""
complevel="3"
varbose=""
declare -a excludes=()

usage() {
  echo "Usage:"
  echo "  ${progname} [OPTIONS] files"
  echo
  echo "Options:"
  echo "  -h, --help"
  echo "  -#: compression level, 1-19 (default: ${complevel})"
  echo "  -P, --processes NUMBER: Maximum number of processes (default: ${procs})"
  echo "  -o FILENAME: output file (default: ${outfile})"
  echo "  -v: verbose mode"
  echo "  -W: overwrite if output file already exists"
  echo "  -x PATTERN:  no compress file pattern"
  echo
  exit 1
}

: "parse options" && {
  while [ -n "$1" ]
  do
    OPT=$1
    case $OPT in
      '-h' | '--help' )
        usage
        ;;
      -[1-9] | -1[0-9] )
        complevel=${OPT:1}
        shift
        ;;
      '-P' | '--processes' )
        if [[ -z $2 ]] || [[ $2 =~ ^-+ ]] ; then
          echo "${progname}: option requires an argument -- $1" 1>&2
          exit 1
        fi
        procs=$2
        shift 2
        ;;
      '-o' )
        if [[ -z $2 ]] || [[ $2 =~ ^-+ ]] ; then
          echo "${progname}: option requires an argument -- $1" 1>&2
          exit 1
        fi
        outfile=$2
        shift 2
        ;;
      '-v' )
        verbose="yes"
        shift
        ;;
      '-W' )
        overwrite="yes"
        shift
        ;;
      '-x' )
        if [[ -z $2 ]] || [[ $2 =~ ^-+ ]] ; then
          echo "${progname}: option requires an argument -- $1" 1>&2
          exit 1
        fi
        excludes+=( "$2" )
        shift 2
        ;;
      -*)
        echo "${progname}: illegal option -- '$(echo $1 | sed 's/^-*//')'" 1>&2
        exit 1
        ;;
      *)
        ((++argc))
        argv+=( "$1" )
        shift
        ;;
    esac
  done

  if [ -z $argv ]; then
    echo "${progname}: too few arguments" 1>&2
    usage
  fi
}

: "create excludes parameter" && {
  excludes_param=()
  for exclude in "${excludes[@]}"
  do
    excludes_param+=("-not -name \"${exclude}\"")
  done
}

: "check output zip file and overwrite" && {
  if [ -n $overwrite ] && [ -f $outfile ] ; then
    echo "${progname}: output file already exists. remove ${outfile}"
    rm ${outfile}
  fi
}

: "main" && {
  echo "${progname}: start compression"
  echo "${progname}: Maximum number of Processes: ${procs}"

  compress_dir() {
    DIR=$1
    echo "${progname}: [Directory: ${DIR}]"
    files=$( eval "find ${DIR} -type f ${excludes_param[@]} -print0" | base64)
    echo ${files} | base64 -D | xargs ${verbose:+-t} -0 -n 1 -P ${procs} zstd -${complevel:-3}
    echo ${files} | base64 -D | xargs ${verbose:+-t} -0 -n 1 -P 1 -I {} zip -0r ${outfile} {}.zst
    echo ${files} | base64 -D | xargs ${verbose:+-t} -0 -n 1 -P ${procs} -I {} rm {}.zst
    echo
  }

  compress_file() {
    FILE=$1
    echo "${progname}: [File: ${FILE}]"
    if [ -z $( eval "find ${FILE} ${excludes_param[@]}" ) ]; then
      zip -0r ${verbose:+-v} ${outfile} ${FILE}
    else
      zstd -${complevel:-3} ${FILE}
      zip -0r ${verbose:+-v} ${outfile} ${FILE}.zst
      rm ${FILE}.zst
    fi
    echo
  }

  for file in "${argv[@]}"
  do
    if [ -f $file ]; then
      compress_file $file
    fi
    if [ -d $file ]; then
      compress_dir $file
    fi
  done
}