Skip to content

Instantly share code, notes, and snippets.

@m0rb
Last active October 8, 2021 19:20
Show Gist options
  • Save m0rb/98e5037419ea62a24c62a3059192499a to your computer and use it in GitHub Desktop.
Save m0rb/98e5037419ea62a24c62a3059192499a to your computer and use it in GitHub Desktop.
dumb bash download accelerator
#!/usr/bin/env bash
# dumb parallel downloader
# 10/7/21 morb_at_misentropic_dot_commercial
declare -a FL
CH=16
MP=4
for each in $*
do
case "$1" in
-c|-C|--chunks)
CH=$2
shift 2;
;;
-p|-P|--parallel)
MP=$2
shift 2;
;;
-r|-R|--rate)
RATE="--limit-rate $2"
shift 2;
;;
-?|--*)
echo fail && exit 1
;;
esac
done
FN=$( basename $@ )
TF=$( mktemp )
CL=$( curl -Is "$@" | awk '/nt-[lL]e/{print $2}' | sed 's#\r##' )
[ -z "$CL" ] && echo "failed to HEAD url or no content-length" && exit 1
[ $MP -gt $CH ] && echo nope && exit 1
C=$(( $CL / $CH ))
S=( $( seq 0 $C $CL ) )
I=$(( ${#S[@]} - 1 ))
A=0;
echo "Size: $CL, running $MP children on $I fileparts..."
for i in $(seq 1 ${I})
do
echo "curl -s -o - -r ${A}-${S[$i]} ${RATE} $@ -- | pv -c -N $i > ${FN}.part${i}"
FL+=( "${FN}.part${i}" )
A=$(( ${S[$i]} + 1 ))
done >> $TF
xargs -P${MP} -a $TF -I{} -- sh -c "eval {}"
cat ${FL[@]} > ${FN} && rm ${FL[@]} && rm ${TF}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment