Skip to content

Instantly share code, notes, and snippets.

@DasLampe
Created August 28, 2013 22:14
Show Gist options
  • Save DasLampe/6372037 to your computer and use it in GitHub Desktop.
Save DasLampe/6372037 to your computer and use it in GitHub Desktop.
Small script to download multiple files (e.g. series), when you only have an directory listing from an apache.
#!/bin/bash
#############################################
# license: CC-NC http://www.tldrlegal.com/l/CC-NC
# author: DasLampe <daslampe@lano-crew.org>
# Description: Small script to download multiple files (e.g. series), when you only have an directory Listing.
# This script search recursivly for files without text/html mime-type & download this.
# If required create folders to arrange files.
#############################################
url=$1
function create_dir() {
new_dir=`echo $1 | rev | cut -d \/ -f 2 | rev | sed 's/%20/_/'`
current_dir=`pwd | rev | cut -d \/ -f 1 | rev`
if [ "$parent_dir" != "$current_dir" ]; then
echo "Try to create new dir $new_dir"
mkdir -p $new_dir
cd $new_dir
fi
}
function download() {
file_type=`curl -I $1 | grep "Content-Type: text/html"`
if [ -z "$file_type" ]
then
echo "Download file"
curl -O $1 -silent
else
echo "No file ... search for file to download"
search_file $1
fi
}
function search_file() {
child=`curl $1 | grep -v '<a href="../">' | grep '<a href' | grep -o '<a href=['"'"'"][^"'"'"']*['"'"'"]' | sed -e 's/^<a href=["'"'"']//' -e 's/["'"'"']$//'`
for i in $child
do
create_dir $1$i
download $1$i
done
#Leave dir. Don't delete line!!!!!! ;)
cd ..
}
create_dir $url
download $url
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment