Skip to content

Instantly share code, notes, and snippets.

@mauron85
Last active December 15, 2018 11:09
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save mauron85/ccf66c8f1f303114ff64ae50aaa7758b to your computer and use it in GitHub Desktop.
Save mauron85/ccf66c8f1f303114ff64ae50aaa7758b to your computer and use it in GitHub Desktop.
Resume curl download with filename detection and resume on libssl error
# Retries a command a with backoff.
#
# The retry count is given by ATTEMPTS (default 5), the
# initial backoff timeout is given by TIMEOUT in seconds
# (default 1.)
#
# Successive backoffs double the timeout.
#
# Beware of set -e killing your whole script!
#
# https://coderwall.com/p/--eiqg/exponential-backoff-in-bash
function with_backoff {
local max_attempts=${ATTEMPTS-5}
local timeout=${TIMEOUT-1}
local attempt=0
local exitCode=0
while [[ $attempt < $max_attempts ]]
do
"$@"
exitCode=$?
if [[ $exitCode == 0 ]]
then
break
fi
echo "Failure! Retrying in $timeout.." 1>&2
sleep $timeout
attempt=$(( attempt + 1 ))
timeout=$(( timeout * 2 ))
done
if [[ $exitCode != 0 ]]
then
echo "You've failed me for the last time! ($@)" 1>&2
fi
return $exitCode
}
urldecode() {
# urldecode <string>
local url_encoded="${1//+/ }"
printf '%b' "${url_encoded//%/\\x}"
}
#https://stackoverflow.com/a/26500519/3896616
resume() {
local url=${1:?"Error: Missing url"}
local file=${2}
# try to detect file name from http headers
if [[ -z "$file" ]]; then
local header="$(curl -sI "$url" | tr -d '\r')"
file="$(echo "$header" | sed -nE "s/.*filename.?=(.*'')?(.*)$/\2/p")"
if [[ -z "$file" ]]; then
file="$(echo "$header" | grep -o -E 'Location:.*$')"
if [[ -n "$file" ]]; then
file=$(basename "${file#Location\:}")
fi
file=$(urldecode $file)
fi
fi
# fallback to filename from url
if [[ -z "$file" ]]; then
file="$(basename $1 | cut -d \? -f 1)"
file=$(urldecode $file)
fi
with_backoff curl --speed-limit 0 --speed-limit 60 -C - -o "$file" "$url"
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment