Last active
October 8, 2021 19:20
-
-
Save m0rb/98e5037419ea62a24c62a3059192499a to your computer and use it in GitHub Desktop.
dumb bash download accelerator
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
# dumb parallel downloader | |
# 10/7/21 morb_at_misentropic_dot_commercial | |
declare -a FL | |
CH=16 | |
MP=4 | |
for each in $* | |
do | |
case "$1" in | |
-c|-C|--chunks) | |
CH=$2 | |
shift 2; | |
;; | |
-p|-P|--parallel) | |
MP=$2 | |
shift 2; | |
;; | |
-r|-R|--rate) | |
RATE="--limit-rate $2" | |
shift 2; | |
;; | |
-?|--*) | |
echo fail && exit 1 | |
;; | |
esac | |
done | |
FN=$( basename $@ ) | |
TF=$( mktemp ) | |
CL=$( curl -Is "$@" | awk '/nt-[lL]e/{print $2}' | sed 's#\r##' ) | |
[ -z "$CL" ] && echo "failed to HEAD url or no content-length" && exit 1 | |
[ $MP -gt $CH ] && echo nope && exit 1 | |
C=$(( $CL / $CH )) | |
S=( $( seq 0 $C $CL ) ) | |
I=$(( ${#S[@]} - 1 )) | |
A=0; | |
echo "Size: $CL, running $MP children on $I fileparts..." | |
for i in $(seq 1 ${I}) | |
do | |
echo "curl -s -o - -r ${A}-${S[$i]} ${RATE} $@ -- | pv -c -N $i > ${FN}.part${i}" | |
FL+=( "${FN}.part${i}" ) | |
A=$(( ${S[$i]} + 1 )) | |
done >> $TF | |
xargs -P${MP} -a $TF -I{} -- sh -c "eval {}" | |
cat ${FL[@]} > ${FN} && rm ${FL[@]} && rm ${TF} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment