Skip to content

Instantly share code, notes, and snippets.

@duzun

duzun/downlist

Last active Feb 7, 2020
Embed
What would you like to do?
Download media files from a m3u8 URL
#!/bin/bash
# Download a file listed in a list file
# @author Dumitru Uzun (DUzun.Me)
# @version 1.0.1
_me_="$(basename "$0")"
_start_time_=$(date +%s)
referer=
_cat_=
_dry_run_=0
_hasDest_=1
options=(-f -k -L)
headers=()
# TAB=$(echo -ne "\t")
error() {
while [ $# -gt 0 ]; do
>&2 echo "$1";
shift;
done;
}
usage() {
cat << EOS
Usage: $_me_ [OPTIONS] <url> [<filename>]
Options:
-, --stdout Output the list of segments to stdout
-e, --referer Add referer to curl. Defaults to \`dirname <url>\`
-H, --header Add header to curl, as many as needed
-<curl_flag>
--<curl_option> <option>
EOS
}
abs_url() {
local url=$1
local b=${2:-$base}
b=${b%*/}
case $url in
http://*|https://*|ftp://*|file:///*)
echo "$url"
;;
//*) echo "${b%*://*}:$url" ;;
*) echo "$b/$url" ;;
esac
}
url_path() {
local url=$1
url="$(basename "$url")"
url="${url%%\?*}"
url="${url%%#*}"
echo "$url"
}
ffmetadata() {
ffmpeg -loglevel panic -hide_banner -i "$1" -f ffmetadata "${2:--}"
}
filesize() {
du -b -h "${1:-$dest}" | head -1 | cut -f1
}
elapsed() {
local now
now=$(date +%s)
(( now -= _start_time_ ))
if [ -n "$1" ]; then
date -d@"$now" -u +%T;
else
echo $now;
fi
}
eta() {
local done=$1
local total=${nrSegments:-$2}
local elapsed
elapsed=$(elapsed)
local eta=$(( elapsed * (total - done) / done ))
date -d@$eta -u +%T
}
progress() {
local done=$1
local total=${nrSegments:-$2}
local progress
(( progress = done * 100 / total ))
printf "%3d%%" $progress
}
while [ $# -gt 0 ]; do
key="$1"
case $key in
-|--stdout)
_cat_=1;
shift;
;;
--dry)
_dry_run_=1
shift;
;;
-e|--referer)
referer="$2";
if [ -z "$referer" ]; then
error "Error: $key expects a valid referer URL" ''
exit 1;
fi
shift;shift;
;;
-H|--header)
options+=(-H "$2");
if [ -z "$2" ]; then
error "Error: $key expects a valid header string" ''
exit 1;
fi
shift;shift;
;;
--) break ;;
# any curl option with value
--*)
options+=("$key" "$2")
shift; shift;
;;
# any curl option without value
-*)
options+=("$key")
shift;
;;
# end of options
*) break ;;
esac
done
url="$1"
dest="$2"
dest_dir=
dest_name=
if [ -z "$url" ]; then
usage
exit 1;
fi;
base=$(dirname "$url")
[ -z "$referer" ] && referer="$base"
[ -n "$referer" ] && options+=(-e "$referer")
[ ${#headers[@]} -gt 0 ] && options+=("${headers[@]}")
mapfile -t segments < <(curl -s -S "${options[@]}" "$url" | grep -v "^#")
_ecode=$?
nrSegments=${#segments[@]}
if [ -n "$_cat_" ]; then
for i in "${segments[@]}"; do
abs_url "$i"
done
exit $_ecode;
fi
if [ "$nrSegments" -eq 0 ]; then
error "Empty playlist"
exit 2
fi
echo "last segment: $(url_path "${segments[nrSegments-1]}")"
echo "target: $dest"
echo
if [ -n "$dest" ]; then
dest_name="$(basename "$dest")"
dest_dir="$(dirname "$dest")"
else
_hasDest_=
dest_name=$(url_path "$url")
dest_dir="$PWD"
case "${dest_name##*.}" in
m3u8) dest_name="${dest_name%.*}.mp4" ;;
esac
dest="$dest_dir/$dest_name";
i=
b="$dest_name"
while [ -s "$dest" ]; do
dest_name="$(date +%s)${i}_$b";
dest="$dest_dir/$dest_name";
(( i += 1 ))
done
fi;
if [ "$_dry_run_" -eq 0 ]; then
dest_tmp="$dest_dir/.$dest_name"
dest_last="$dest_dir/.$dest_name.seg"
dest_seg="$dest_dir/.$dest_name.ts"
_resume_=
c=0
if [ -s "$dest_tmp" ]; then
_resume_=$(cat "$dest_last")
idx=
for i in "${segments[@]}"; do
(( c += 1 ))
j=$(url_path "$i")
if [ "$j" == "$_resume_" ]; then
idx=$c
break
fi
done
if [ -n "$idx" ]; then
echo "resume at $idx/$nrSegments $(url_path "$i")"
segments=("${segments[@]:$idx:$nrSegments}")
else
_resume_=
c=0
fi
fi
if [ -z "$_resume_" ]; then
echo -n > "$dest_tmp"
fi
for i in "${segments[@]}"; do
(( c += 1 ))
echo "$c/$nrSegments $(progress $c) size $(filesize "$dest_tmp") elapsed $(elapsed 1) left $(eta $c) $(url_path "$i")";
u=$(abs_url "$i")
if ! curl -o "$dest_seg" -# "${options[@]}" --retry 7 --retry-delay 0 --connect-timeout 7 --max-time 10 "$u"; then
_ecode=$?;
error "Error :( $_ecode @ $i"
exit 3;
fi
cat "$dest_seg" >> "$dest_tmp" && \
url_path "$i" > "$dest_last"
echo -ne "\033[2A"
done;
rm -f -- "$dest_last" "$dest_seg"
echo
echo
echo
if [ -s "$dest_tmp" ]; then
if command -v ffmpeg > /dev/null; then
echo "Processing with ffmpeg...";
echo
ffmetadata "$dest_tmp"
echo
ffmpeg -loglevel panic -hide_banner -y -i "$dest_tmp" -c copy -strict experimental "$dest" && \
rm -f -- "$dest_tmp"
else
echo "ffmpeg not found"
mv -f -- "$dest_tmp" "$dest"
fi
fi
fi
echo "done in $(elapsed 1)";
echo
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment