public
Last active

pararell-curl.sh

  • Download Gist
pararell-curl.sh
Shell
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108
#!/bin/bash
#
# $ pararell-curl.sh --url=[url]
# $ pararell-curl.sh --url=[url] --thread=6
#
LANG=C
LC_ALL=C
 
set -e
 
opts=""
# extract opts
for arg in $*; do
case $arg in
--*=*)
key=${arg%%=*}; key=${key##--}
value=${arg##--*=}
eval ${key}=${value}
opts="${opts} ${key}"
;;
esac
done
 
url=${url:-http://ftp.riken.jp/Linux/centos/6.0/isos/i386/CentOS-6.0-i386-bin-DVD.iso}
thread=${thread:-6}
tmp_path=${tmp_path:-/var/tmp/__$(basename $0)}
part_name=$(basename ${url})
output_dir=${output_dir:-${tmp_path}}
output_path=${output_path:-${output_dir}/${part_name}}
retry=${retry:-3}
 
case ${url} in
http://*|https://*)
content_length=$(curl --retry ${retry} -s -L --head ${url} | egrep ^Content-Length | awk '{print $2}' | strings)
;;
file:///*)
content_length=$(ls -l ${url##file://} | awk '{print $5}')
;;
*)
[ -f ${url} ] && {
content_length=$(ls -l ${url} | awk '{print $5}')
url="file://${url}"
} || {
echo not supported scheme. >&2
exit 1
}
;;
esac
 
[ -z "${content_length}" ] && { exit 0; }
[ ${thread} -ge ${content_length} ] && thread=1
 
range=$((${content_length} / ${thread}))
parts=
 
echo content-length: ${content_length} / ${thread}
 
pids=
trap 'kill -9 ${pids};' 2
 
function shlog {
echo "\$ $*"
eval $*
}
 
[ -d ${tmp_path} ] || mkdir -p ${tmp_path}
[ -d ${output_dir} ] || mkdir -p ${output_dir}
 
cur=0
while [ ${cur} -lt ${thread} ]; do
from=$((${range} * ${cur}))
if [ ${cur} = $((${thread} -1 )) ]; then
to=
else
to=$((${range} * $((${cur} + 1)) - 1))
fi
 
part_path=${tmp_path}/${part_name}.${cur}
shlog "curl --retry ${retry} -s -L --range ${from}-${to} -o ${part_path} ${url} &"
pids="${pids} $!"
 
parts="${parts} ${part_path}"
 
cur=$((${cur} + 1))
done
 
echo wait: ${pids}
wait ${pids}
 
echo "concat parts..."
cat ${parts} > ${output_path}
 
for part in ${parts}; do
[ -f ${part} ] && rm -f ${part}
done
 
sync
 
generated_length=$(ls -l ${output_path} | awk '{print $5}')
[ ${content_length} = ${generated_length} ] || {
echo "no mutch file size" >&2
echo "content_length: ${content_length} != ${generated_length}" >&2
[ -f ${output_path} ] && rm -f ${output_path}
exit 1
}
 
echo "=> ${output_path}"
ls -l ${output_path}

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.