Last active
December 15, 2018 11:09
-
-
Save mauron85/ccf66c8f1f303114ff64ae50aaa7758b to your computer and use it in GitHub Desktop.
Resume curl download with filename detection and resume on libssl error
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Retries a command a with backoff. | |
# | |
# The retry count is given by ATTEMPTS (default 5), the | |
# initial backoff timeout is given by TIMEOUT in seconds | |
# (default 1.) | |
# | |
# Successive backoffs double the timeout. | |
# | |
# Beware of set -e killing your whole script! | |
# | |
# https://coderwall.com/p/--eiqg/exponential-backoff-in-bash | |
function with_backoff { | |
local max_attempts=${ATTEMPTS-5} | |
local timeout=${TIMEOUT-1} | |
local attempt=0 | |
local exitCode=0 | |
while [[ $attempt < $max_attempts ]] | |
do | |
"$@" | |
exitCode=$? | |
if [[ $exitCode == 0 ]] | |
then | |
break | |
fi | |
echo "Failure! Retrying in $timeout.." 1>&2 | |
sleep $timeout | |
attempt=$(( attempt + 1 )) | |
timeout=$(( timeout * 2 )) | |
done | |
if [[ $exitCode != 0 ]] | |
then | |
echo "You've failed me for the last time! ($@)" 1>&2 | |
fi | |
return $exitCode | |
} | |
urldecode() { | |
# urldecode <string> | |
local url_encoded="${1//+/ }" | |
printf '%b' "${url_encoded//%/\\x}" | |
} | |
#https://stackoverflow.com/a/26500519/3896616 | |
resume() { | |
local url=${1:?"Error: Missing url"} | |
local file=${2} | |
# try to detect file name from http headers | |
if [[ -z "$file" ]]; then | |
local header="$(curl -sI "$url" | tr -d '\r')" | |
file="$(echo "$header" | sed -nE "s/.*filename.?=(.*'')?(.*)$/\2/p")" | |
if [[ -z "$file" ]]; then | |
file="$(echo "$header" | grep -o -E 'Location:.*$')" | |
if [[ -n "$file" ]]; then | |
file=$(basename "${file#Location\:}") | |
fi | |
file=$(urldecode $file) | |
fi | |
fi | |
# fallback to filename from url | |
if [[ -z "$file" ]]; then | |
file="$(basename $1 | cut -d \? -f 1)" | |
file=$(urldecode $file) | |
fi | |
with_backoff curl --speed-limit 0 --speed-limit 60 -C - -o "$file" "$url" | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment