Skip to content

Instantly share code, notes, and snippets.

@imaya
Last active July 22, 2017 13:28
Show Gist options
  • Save imaya/1626204dc0853a8a1cf8b44cf3822419 to your computer and use it in GitHub Desktop.
Save imaya/1626204dc0853a8a1cf8b44cf3822419 to your computer and use it in GitHub Desktop.
事前に zstd にして zip ファイルをコンテナとして扱うやつ

zipstd

Zstandard compression in zip.

Install

$ make test
$ INSTALL_DIR=[zipstd install directory] make install

Usage

Compression

Usage:
  zipstd [OPTIONS] files

Options:
  -h, --help
  -#: compression level, 1-19 (default: 3)
  -P, --processes NUMBER: Maximum number of processes (default: 1)
  -o FILENAME: output file (default: out.zip)
  -v: verbose mode
  -W: overwrite if output file already exists
  -x PATTERN:  no compress file pattern

decompression

Usage:
  unzipstd [OPTIONS] files

Options:
  -h, --help
  -P, --processes ARG: Maximum number of processes (default: 1)
  -o ARG: output directory (default: .)
  -v: verbose mode

License

MIT

test:
which /bin/bash || { echo "please install bash to /bin/bash"; exit 1 ; }
which base64 || { echo "please install base64"; exit 1 ; }
which find || { echo "please install find"; exit 1 ; }
which sed || { echo "please install sed"; exit 1 ; }
which unzip || { echo "please install unzip"; exit 1 ; }
which xargs || { echo "please install xargs"; exit 1 ; }
which zip || { echo "please install zip"; exit 1 ; }
which zstd || { echo "please install zstd"; exit 1 ; }
install: test
[ -n "${INSTALL_DIR}" ] || { echo "Usage:\n\tINSTALL_DIR=Directory make install"; exit 1 ; }
[ -d "${INSTALL_DIR}" ] || { echo "${INSTALL_DIR} not exists"; exit 1 ; }
install zipstd.sh "${INSTALL_DIR}/zipstd"
install unzipstd.sh "${INSTALL_DIR}/unzipstd"
#!/bin/bash
declare -i argc=0
declare -a argv=()
progname=$(basename $0)
outdir="."
procs=1
varbose=""
usage() {
echo "Usage:"
echo " ${progname} [OPTIONS] files"
echo
echo "Options:"
echo " -h, --help"
echo " -P, --processes ARG: Maximum number of processes (default: ${procs})"
echo " -o ARG: output directory (default: ${outdir})"
echo " -v: verbose mode"
echo
exit 1
}
: "parse options" && {
while [ -n "$1" ]
do
OPT=$1
case $OPT in
'-h' | '--help' )
usage
;;
'-P' | '--processes' )
if [[ -z $2 ]] || [[ $2 =~ ^-+ ]] ; then
echo "${progname}: option requires an argument -- $1" 1>&2
exit 1
fi
procs=$2
shift 2
;;
'-o' )
if [[ -z $2 ]] || [[ $2 =~ ^-+ ]] ; then
echo "${progname}: option requires an argument -- $1" 1>&2
exit 1
fi
outdir=$2
shift 2
;;
'-v' )
verbose="yes"
shift
;;
-*)
echo "${progname}: illegal option -- '$(echo $1 | sed 's/^-*//')'" 1>&2
exit 1
;;
*)
((++argc))
argv+=( "$1" )
shift
;;
esac
done
if [ -z $argv ]; then
echo "${progname}: too few arguments" 1>&2
usage
fi
}
: "main" && {
echo "${progname}: start decompression"
echo "${progname}: Maximum number of Processes: ${procs}"
decompress() {
file=$1
echo "[File: ${file}]"
unzip ${file} -d ${outdir}
find ${outdir} -type f -name "*.zst" -print0 | xargs ${verbose:+-t} -0 -n 1 -P ${procs} zstd -d --rm
echo
}
for file in ${argv[@]}
do
if [ -f $file ]; then
decompress $file
fi
if [ -d $file ]; then
echo "${progname}: skip. ${file} is directory"
fi
done
}
#!/bin/bash
declare -i argc=0
declare -a argv=()
progname=$(basename $0)
outfile="out.zip"
srcdir=$1
procs=1
overwrite=""
complevel="3"
varbose=""
declare -a excludes=()
usage() {
echo "Usage:"
echo " ${progname} [OPTIONS] files"
echo
echo "Options:"
echo " -h, --help"
echo " -#: compression level, 1-19 (default: ${complevel})"
echo " -P, --processes NUMBER: Maximum number of processes (default: ${procs})"
echo " -o FILENAME: output file (default: ${outfile})"
echo " -v: verbose mode"
echo " -W: overwrite if output file already exists"
echo " -x PATTERN: no compress file pattern"
echo
exit 1
}
: "parse options" && {
while [ -n "$1" ]
do
OPT=$1
case $OPT in
'-h' | '--help' )
usage
;;
-[1-9] | -1[0-9] )
complevel=${OPT:1}
shift
;;
'-P' | '--processes' )
if [[ -z $2 ]] || [[ $2 =~ ^-+ ]] ; then
echo "${progname}: option requires an argument -- $1" 1>&2
exit 1
fi
procs=$2
shift 2
;;
'-o' )
if [[ -z $2 ]] || [[ $2 =~ ^-+ ]] ; then
echo "${progname}: option requires an argument -- $1" 1>&2
exit 1
fi
outfile=$2
shift 2
;;
'-v' )
verbose="yes"
shift
;;
'-W' )
overwrite="yes"
shift
;;
'-x' )
if [[ -z $2 ]] || [[ $2 =~ ^-+ ]] ; then
echo "${progname}: option requires an argument -- $1" 1>&2
exit 1
fi
excludes+=( "$2" )
shift 2
;;
-*)
echo "${progname}: illegal option -- '$(echo $1 | sed 's/^-*//')'" 1>&2
exit 1
;;
*)
((++argc))
argv+=( "$1" )
shift
;;
esac
done
if [ -z $argv ]; then
echo "${progname}: too few arguments" 1>&2
usage
fi
}
: "create excludes parameter" && {
excludes_param=()
for exclude in "${excludes[@]}"
do
excludes_param+=("-not -name \"${exclude}\"")
done
}
: "check output zip file and overwrite" && {
if [ -n $overwrite ] && [ -f $outfile ] ; then
echo "${progname}: output file already exists. remove ${outfile}"
rm ${outfile}
fi
}
: "main" && {
echo "${progname}: start compression"
echo "${progname}: Maximum number of Processes: ${procs}"
compress_dir() {
DIR=$1
echo "${progname}: [Directory: ${DIR}]"
files=$( eval "find ${DIR} -type f ${excludes_param[@]} -print0" | base64)
echo ${files} | base64 -D | xargs ${verbose:+-t} -0 -n 1 -P ${procs} zstd -${complevel:-3}
echo ${files} | base64 -D | xargs ${verbose:+-t} -0 -n 1 -P 1 -I {} zip -0r ${outfile} {}.zst
echo ${files} | base64 -D | xargs ${verbose:+-t} -0 -n 1 -P ${procs} -I {} rm {}.zst
echo
}
compress_file() {
FILE=$1
echo "${progname}: [File: ${FILE}]"
if [ -z $( eval "find ${FILE} ${excludes_param[@]}" ) ]; then
zip -0r ${verbose:+-v} ${outfile} ${FILE}
else
zstd -${complevel:-3} ${FILE}
zip -0r ${verbose:+-v} ${outfile} ${FILE}.zst
rm ${FILE}.zst
fi
echo
}
for file in "${argv[@]}"
do
if [ -f $file ]; then
compress_file $file
fi
if [ -d $file ]; then
compress_dir $file
fi
done
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment