BASH helpers / backports / shims

_bash_script_template.sh

#!/usr/bin/env bash
set -euo pipefail

IFS=$'\n\t'
SCRIPT_DIR=$(realpath -- $(dirname -- $0))
SCRIPT_NAME=$(basename "$0")
#SCRIPT_TMP=$(mktemp -d "${TMPDIR:-/tmp}/${SCRIPT_NAME}_XXXXXXX")
SCRIPT_VERSION='0.0.1'
VERBOSITY=0
TARGET_USER=

if [ -n "${TARGET_USER}" ]; then
    # Check if the script is already running as the target user
    if [ "$(id -u)" -eq "$(id -u "${TARGET_USER}")" ]; then
        echo "Running as ${TARGET_USER}. Continuing with the script." >&2
    else
        # Check if the script is running as root
        if [ "$(id -u)" -eq 0 ]; then
            echo "Running as root. Changing to user: ${TARGET_USER}." >&2
            # Switch to the target user and pass along command-line arguments
            su "${TARGET_USER}" -c /usr/bin/env bash "$(readlink -f "$0")" $@
            exit $?
        else
            echo "Not running as root or ${TARGET_USER}" >&2
            exit 1
        fi
    fi
fi

print_usage(){
cat <<EOF
Usage: ${SCRIPT_NAME} [OPTIONS] -- file
    -[v]+  --verbose=[0-9]            Set the verbosity of the output
    -V     --version                  Get the current Version
    -h     --help                     Print this help screen
EOF
}


if [ $# -eq 0 ]; then
    echo 'Error: need at least one parameter' >&2
    print_usage >&2
    exit 1
fi

# no colon - option doesn't take any arguments.
# :        - option must have a required argument.
# ::       - option can have an optional argument.
OPTS=$(getopt -o 'vVh' -l 'verbose,version,help' -n "${SCRIPT_NAME}" -- "$@")

eval set -- "$OPTS"
while true; do
    echo "OPTION: ${1}"
    case "$1" in
        -v|--verbose)
            VERBOSITY=$(( ${VERBOSITY} + 1 ))
            shift
            ;;
        -V|--version)
            echo "${SCRIPT_NAME} ${SCRIPT_VERSION}"
            exit 0
            ;;
        -h|--help)
            print_usage
            exit 0
            ;;
        --) shift; break ;; # stop processing options
        *)
            echo "ERROR: Invalid option $1" >&2
            print_usage >&2
            exit 1
        ;;
    esac
done
if [ "${VERBOSITY}" -ge 1 ]; then
    set -x
fi

echo $@
echo "VERBOSITY: ${VERBOSITY}"

abspath.sh

# realpath isn't a part of coreutils
# src: https://github.com/ko1nksm/readlinkf
# realpath vs readlink -f https://unix.stackexchange.com/a/136527
abspath(){
    [ ${1:+x} ] || return 1; p=$1; until [ _"${p%/}" = _"$p" ]; do p=${p%/}; done
    [ -e "$p" ] && p=$1; [ -d "$1" ] && p=$p/; set 10 "$(pwd)" "${OLDPWD:-}"; PWD=
    CDPATH="" cd -P "$2" && while [ "$1" -gt 0 ]; do set "$1" "$2" "$3" "${p%/*}"
    [ _"$p" = _"$4" ] || { CDPATH="" cd -P "${4:-/}" || break; p=${p##*/}; }
    [ ! -L "$p" ] && p=${PWD%/}${p:+/}$p && set "$@" "${p:-/}" && break
    set $(($1-1)) "$2" "$3" "$p"; p=$(ls -dl "$p") || break; p=${p#*" $4 -> "}
    done 2>/dev/null; cd "$2" && OLDPWD=$3 && [ ${5+x} ] && printf '%s\n' "$5"
}

bad_service.sh

#!/usr/bin/env bash
set -euo pipefail
# set -x

export TOOL_PATH="$HOME/bin/example"
export TOOL_NAME="$(basename ${TOOL_PATH})"
export ARGUMENTS=""

if pgrep "${TOOL_NAME}" &>/dev/null ; then
    echo -en "\e[32m${TOOL_NAME} is running, do you want to restart it? yes/no \e[0m"
    read -p ":" reply
    if [[ "$reply" = 'yes' ]]; then
        echo -e "\e[33mrestarting ${TOOL_NAME}...\e[0m"
        screen -X -S "${TOOL_NAME}" quit >&/dev/null || true
        screen -wipe >&/dev/null
        screen -dm -S "${TOOL_NAME}" ${TOOL_PATH} ${ARGUMENTS}
        sleep 2
        pgrep "${TOOL_NAME}" >&/dev/null
        [[ $? = 0 ]] && echo -e "\e[31m${TOOL_NAME} restarted and is running\e[0m" || echo -e "\e[31mcan't start ${TOOL_NAME}\e[0m"
    else
        echo -e "\e[31mexiting...\e[0m"
    fi
else
    echo -e "\e[33mstarting ${TOOL_NAME}...\e[0m"
    screen -X -S "${TOOL_NAME}" quit >&/dev/null || true
    screen -wipe >&/dev/null
    screen -dm -S "${TOOL_NAME}" ${TOOL_PATH} ${ARGUMENTS}
    sleep 2
    pgrep "${TOOL_NAME}" >&/dev/null
    [[ $? = 0 ]] && echo -e "\e[31m${TOOL_NAME} restarted and is running\e[0m" || echo -e "\e[31mcan't start ${TOOL_NAME}\e[0m"
fi

fancy_permissions.sh

TARGETDIR=$PWD

chown -R user:staff ${TARGETDIR}
chmod -R u=rwX,g=rwX,o=rX ${TARGETDIR}
find ${TARGETDIR} -type d -exec chmod 2775 {} \;
setfacl -d -m mask:007 ${TARGETDIR} # aka. umask 002 for this folder

# remove all ACLs
setfacl -R -b ${TARGETDIR} 
find . -type d -exec chmod 00755 {} \;
find . -type f -exec chmod 00644 {} \;

multi-curl.sh

#!/bin/bash
# src: https://github.com/WanghongLin/miscellaneous/blob/master/tools/mcurl.sh

slices=20

case $OSTYPE in
    *linux*) slices=$(grep -c processor /proc/cpuinfo) ;;
    *darwin*) slices=$(sysctl hw.ncpu | cut -d' ' -f2) ;;
    *cygwin*) slices=$NUMBER_OF_PROCESSORS ;;
    *) slices=20 ;;
esac

url=
output=
curl_options=

function usage ()
{
    echo "Usage :  $0 [options] url

    Options:
    -h|help       Display this message
    -s|slice      How many slices the download task will split, default is $slices
    -o|output     Specify the output file name, use the guessing file name from url as output file name if not specify this option"

}

while getopts ":hv:s:o:" opt
do
    case $opt in
		h|help     )  usage; exit 0   ;;
		s|slice    )  slices=$OPTARG ;;
		o|output   )  output=$OPTARG ;;
		* )  echo -e "\n  Option does not exist : $OPTARG\n"
	    	usage; exit 1   ;;
    esac
done
shift $(($OPTIND-1))

url=${@: -1}

if ! [[ $url =~ ^https?://.*$ ]];then
    printf "\e[31mInvalid URL $url\e[0m\n"
    usage
    exit 1
fi

url_no_query=${url%%\?*}
file_to_save=${url_no_query##*/}

[ x$output != x ] && file_to_save=$output

echo "Download $url to $file_to_save with $slices tasks."

size_in_byte=$(curl -I "$url" "$curl_options" 2>/dev/null | sed -n 's/\([Cc]ontent-[Ll]ength:\)\(.*\)/\2/p' | tr -d [[:space:]])

if ! [[ $size_in_byte =~ ^[0-9]+$ ]];then
    printf "\e[31mCould not get content length, make sure your resource have content length response.\e[0m\n"
    exit 1
fi

size_per_slice=$(($size_in_byte/$slices))
let size_per_slice=${size_per_slice}+1  # avoid rounding issue

total_slice=${slices}
finished_slice=0
is_finished=0
function callback()
{
	subp=$(pgrep -P $$ | wc -l)
	if [  $subp -eq 1 ];then
		for s in `seq $total_slice`
		do
			cat $$.$s >> "${file_to_save}"
			rm $$.$s
		done
		is_finished=1
	fi
}

function run()
{
	curl -r $2-$3 $url -o $1 2>/dev/null && kill -n 10 $$ &
}

trap callback 10

start_time=$(date +%s)
for s in `seq $total_slice`; do
	begin=$((($s-1)*${size_per_slice}))
	if [ $begin -ne 0 ];then
		begin=$((begin+=1))
	fi
	end=$(($s*$size_per_slice))
	if [ $end -gt $size_in_byte ];then
		end=
	fi
	run $$.$s $begin $end
done

until [ $is_finished -eq 1 ]; do
	if [ -f $$.1 ];then
		total_kb=$(BLOCKSIZE=1024 du -k $$.* | awk '{t+=$1}END{printf "%d", t}')
		duration=$((`date +%s`-$start_time))
		[ $duration -gt 0 ] && printf "\rCurrent average speed %4d KiB/s" $(($total_kb/$duration))
	fi
	sleep 1
done

echo

powershell

#!/usr/bin/env bash

if [ $# -eq 0 ]; then
    /mnt/c/Windows/System32/WindowsPowerShell/v1.0/powershell.exe
elif [ $# -eq 1 ]; then
    psscriptpath="$(wslpath -w $1)"
    /mnt/c/Windows/System32/WindowsPowerShell/v1.0/powershell.exe -File "${psscriptpath}"
else
    /mnt/c/Windows/System32/WindowsPowerShell/v1.0/powershell.exe $@
fi

readarray.sh

if ! command -v readarray >/dev/null && ! command -v mapfile >/dev/null; then
    # src: https://stackoverflow.com/a/64793921
    # man: https://ss64.com/bash/mapfile.html
    # Very minimal readarray implementation using read. Does NOT work with lines that contain double-quotes due to eval()
    readarray() {
        local cmd opt t v=MAPFILE
        while [ -n "$1" ]; do
            case "$1" in
            -h|--help) echo "minimal substitute mapfile for older bash"; return; ;;
            -r) shift; opt="$opt -r"; ;;
            -t) shift; t=1; ;;
            -u)
                shift;
                if [ -n "$1" ]; then
                  opt="$opt -u $1";
                  shift
                fi
                ;;
            *)
                if [[ "$1" =~ ^[A-Za-z_]+$ ]]; then
                    v="$1"
                    shift
                else
                    echo "Error: Unknown option: '$1'" 1>&2
                    return 1
                fi
                ;;
            esac
        done
        cmd="read $opt"
        eval "$v=()"
        while IFS= eval "$cmd line"; do
            line=$(echo "$line" | sed -e "s#\([\"\`]\)#\\\\\1#g" )
            eval "${v}+=(\"$line\")"
        done
    }
    mapfile() {
        readarray $@
    }
fi

rsync-between.sh

#!/usr/bin/env bash
# https://unix.stackexchange.com/a/183516

if [[ "$1" =~ ^([^@]+@)?([^:]+):(.*)$ ]]; then
  src_ssh_user="${BASH_REMATCH[1]}"
  src_host="${BASH_REMATCH[2]}"
  src_dir="${BASH_REMATCH[3]}"
fi

if [[ "$2" =~ ^([^@]+@)?([^:]+):(.*)$ ]]; then
  target_ssh_user="${BASH_REMATCH[1]}"
  target_host="${BASH_REMATCH[2]}"
  target_dir="${BASH_REMATCH[3]}"
fi

# for varname in src_ssh_user src_host src_dir target_ssh_user target_host target_dir; do
#   echo "${!varname@A}"
# done

rsync_opts="${3:--arv -l --info=progress2 --rsync-path=\"sudo rsync\" --dry-run}"

if [  -z $src_host ] || [ -z $src_dir ] || [ -z $target_host ] || [ -z $target_dir ] ; then
    echo "usage: rsync-between src_ssh_user@src-host:src_dir target_ssh_user@target-host:target_dir 'rsync_opts'" >&2
    echo "all parameters are necessary"
    exit 1
fi

# ssh -A -R localhost:50000:<TARGET_HOST>:22 <SRC_HOST> 'sudo --preserve-env=SSH_AUTH_SOCK rsync -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 50000" --rsync-path="sudo rsync" -ar -l --delete --info=progress2 --dry-run <EXTRA_RSYNC_OPTS> <SRC_DIR> ${SUDO_USER}@localhost:<TARGET_DIR>'

printf -v command "ssh -A -R localhost:50000:%s:22 %s%s 'sudo --preserve-env=SSH_AUTH_SOCK rsync -e ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \"ssh -o -p 50000\" %s %slocalhost:%s'" \
    "${target_host}" "${src_ssh_user}" "${src_host}" "${ssh_extra_opts}" "${rsync_opts}" "${src_dir}" "${target_ssh_user}" "${target_dir}"

echo $command

rsync_chunked.sh

# TODO: look at this https://github.com/nathanhaigh/parallel-rsync/blob/main/prsync

# check if Source and Destination is given
if [ "$#" -lt 2 ]; then
    echo "usage: rsync-copy-chunked SRC DEST [OPTION]" 1>&2
    return 1
fi

local SOURCE="$1"
local DESTINATION="$2"
shift 2 # removes entries from the arguments array

if ( echo "$@" | grep "\-j=" >/dev/null ); then
    local RSYNC_THREADS=$(echo $1 | sed -e 's/-j=//g')
    shift
fi

local RSYNC_ARGUMENTS="${RSYNC_ARGUMENTS:-$@}"
local RSYNC_THREADS="${RSYNC_THREADS:-$(nproc --all 2>/dev/null || sysctl -n hw.ncpu || echo 2)}"
# local RSYNC_MAX_RETRIES="${RSYNC_MAX_RETRIES:-5}"

local RSYNC_TMP_DIR="${TMPDIR:-'/tmp/'}rsync-copy-chunked-$(id -u)-$(date +'%s')"
command -v mktemp >/dev/null && local RSYNC_TMP_DIR="$(mktemp -d)"
mkdir -p "${RSYNC_TMP_DIR}"
local TRANSFER_LIST="${RSYNC_TMP_DIR}/transfer_list.txt"
touch "${TRANSFER_LIST}"

if echo "${SOURCE}" | grep ':' >/dev/null; then
     local RSYNC_REMOTE_HOST="$(echo ${SOURCE} | cut -d ':' -f1)"
     local RSYNC_REMOTE_SOURCE="$(echo ${SOURCE} | cut -d ':' -f2)"
     ssh ${RSYNC_REMOTE_HOST} "cd ${RSYNC_REMOTE_SOURCE:-/dev/null} && find . -type f ! \( -name '._*' -o -name '.DS_Store' \)" >| "${TRANSFER_LIST}" && sync

     # export LC_RSYNC_REMOTE_SOURCE="$(echo ${SOURCE} | cut -d ':' -f2)"
     # ssh -o SendEnv=LC_RSYNC_REMOTE_SOURCE ${RSYNC_REMOTE_HOST} 'cd "${LC_RSYNC_REMOTE_SOURCE:-/dev/null}" && find . -type f ! \( -name "._*" -o -name ".DS_Store" \)' >| "${TRANSFER_LIST}" && sync
     # unset LC_RSYNC_REMOTE_SOURCE
else
    local pwd_bkp="${PWD}"
    cd "${SOURCE}" && find . -type f ! \( -name '._*' -o -name '.DS_Store' \) >| "${TRANSFER_LIST}" && sync
    cd "${pwd_bkp}"
fi

local TRANSFER_COUNT="$(wc -l ${TRANSFER_LIST} | awk '{ printf $1}')"
local CHUNK_SIZE=$(echo "${TRANSFER_COUNT} / ${RSYNC_THREADS}" | bc -l | awk '{ printf("%d", $1 + 0.5) }')
local CHUNK_PREFIX=".chunk_"
local START_TIME="$(date +'%s')"
local SESSION="rsync-$(basename ${RSYNC_TMP_DIR})"

echo ""
echo "Summary:"
echo "-----------------"
echo "SOURCE: ${SOURCE}"
echo "DESTINATION: ${DESTINATION}"
echo "RSYNC_THREADS: ${RSYNC_THREADS}"
echo "CHUNK_SIZE: ${CHUNK_SIZE}"
# echo "RSYNC_MAX_RETRIES: ${RSYNC_MAX_RETRIES}"
echo "RSYNC_ARGUMENTS: ${RSYNC_ARGUMENTS}"
echo "RSYNC_TMP_DIR: ${RSYNC_TMP_DIR}"
echo "SESSION: ${SESSION}"
echo ""

sleep 5 # cancel opportuniy

split -l "${CHUNK_SIZE}" "${TRANSFER_LIST}" "${TRANSFER_LIST}${CHUNK_PREFIX}"; sync

local SCREEN_WINDOW_NR=1
screen -dmS "${SESSION}"

for CHUNK in $(find "${RSYNC_TMP_DIR}" -name "*${CHUNK_PREFIX}*"); do
    screen -r "${SESSION}" -X screen -t "chunk-${CHUNK}" # create screen window

    touch "${CHUNK}.in-progress"; sync
    local RSYNC_COMMAND="rsync -avR ${RSYNC_ARGUMENTS} --exclude '.DS_Store' --exclude '._*' --exclude 'Thumbs.db' --exclude 'desktop.ini' --files-from=\"${CHUNK}\" \"${SOURCE}\" \"${DESTINATION}\" 2>&1 | tee \"${CHUNK}.in-progress\" || cp -f \"${CHUNK}.in-progress\" \"${CHUNK}.failed\"; rm -f \"${CHUNK}.in-progress\" ; exit"

    echo "${RSYNC_COMMAND}" >> "${CHUNK}.in-progress"
    echo "--------------------------" >> "${CHUNK}.in-progress"

    # local SESSION_COMMAND=$(echo 'false; while [ $? -ne 0 -a ${TRY:-0} -lt '"${RSYNC_MAX_RETRIES:-5}"' ]; do TRY=\$(echo \"${TRY} + 1\" | bc -l); '"${RSYNC_COMMAND:-debug: \$TRY}"' ; done')

    echo "running: ${CHUNK}.in-progress"

    # stuff is a screen command: https://www.gnu.org/software/screen/manual/screen.html#Paste
    screen -dr "$SESSION" -p "${SCREEN_WINDOW_NR}" -X stuff " ${RSYNC_COMMAND}\n"

    local SCREEN_WINDOW_NR=$(echo "${SCREEN_WINDOW_NR} + 1" | bc -l)
done

echo "all jobs started."
echo "you may cancel this command, the tasks will continue to run in their screen session"

screen -r "${SESSION}" -p 0 -X stuff " exit\n" # close unnecessary startup window

sleep 1 # creating .in-progress files takes time
tail -f ${RSYNC_TMP_DIR}/*.in-progress &
local tail_pid=$!

while [ -n "$(find "${RSYNC_TMP_DIR}" -name "*${CHUNK_PREFIX}*.in-progress")" ]; do
    sleep 1 # loop untill all lock files are gone
done

kill $tail_pid 2>&1 >/dev/null

local END_TIME="$(date +'%s')"
local DURATION=$(echo "${END_TIME} - ${START_TIME}" | bc -l)

echo ""
echo "Finished Summary:"
echo "-----------------"
echo "DURATION: ${DURATION}s"
echo "SOURCE: ${SOURCE}"
echo "DESTINATION: ${DESTINATION}"
echo "RSYNC_THREADS: ${RSYNC_THREADS}"
echo "CHUNK_SIZE: ${CHUNK_SIZE}"
# echo "RSYNC_MAX_RETRIES: ${RSYNC_MAX_RETRIES}"
echo "RSYNC_ARGUMENTS: ${RSYNC_ARGUMENTS}"
echo "RSYNC_TMP_DIR: ${RSYNC_TMP_DIR}"
echo "SESSION: ${SESSION}"

if [ -n "$(find "${RSYNC_TMP_DIR}" -name "*${CHUNK_PREFIX}*.failed")" ]; then
    echo "some sessions failed:"
    find "${RSYNC_TMP_DIR}" -name "*${CHUNK_PREFIX}*.failed"
    return 1
fi

rm -rf "${RSYNC_TMP_DIR}" # cleanup

sourced_or_run.sh

#!/bin/sh

# https://stackoverflow.com/a/28776166

sourced=0
if [ -n "$ZSH_VERSION" ]; then
  case $ZSH_EVAL_CONTEXT in *:file) sourced=1;; esac
elif [ -n "$KSH_VERSION" ]; then
  [ "$(cd -- "$(dirname -- "$0")" && pwd -P)/$(basename -- "$0")" != "$(cd -- "$(dirname -- "${.sh.file}")" && pwd -P)/$(basename -- "${.sh.file}")" ] && sourced=1
elif [ -n "$BASH_VERSION" ]; then
  (return 0 2>/dev/null) && sourced=1
else
  case ${0##*/} in sh|-sh|dash|-dash) sourced=1;; esac
fi

__example_prog(){
  echo "HELLO $@"
}

if [ $sourced -eq 1 ]; then
    alias 'example-prog'='__example_prog'
else
    __example_prog $@
fi