Skip to content

Instantly share code, notes, and snippets.

@T31337
Last active July 6, 2017 19:53
Show Gist options
  • Save T31337/9acfb1b5f1f126ca0854c622aa9b03fe to your computer and use it in GitHub Desktop.
Save T31337/9acfb1b5f1f126ca0854c622aa9b03fe to your computer and use it in GitHub Desktop.
#!/bin/bash
URL="-i sites.txt"
ACCEPT='html,htm,css,js,php,jpg,png,jpeg,gif,bmp,img,ico,svg' #File Accept/Download List
DECLINE='tmp' #File Reject/Decline List
DIR="$HOME/Downloads/web/"
function downloadSite
{
wget --no-clobber --convert-links --random-wait -r -p -E -e robots=off -U mozilla -P $DIR $URL
}
function downlaod
{
wget --no-clobber --convert-links --random-wait -r -p -E -e robots=off -U mozilla -r --retr-symlinks --no-hsts --no-check-certificate --no-parent --auth-no-challenge --no-cookies --ignore-length --no-cache --ignore-case --no-dns-cache --random-wait --no-use-server-timestamps -A "$ACCEPT" -R "$REJECT" --continue -P $DIR $URL
}
function spider
{
wget -p -k -e robots=off -r --spider --retr-symlinks --no-hsts --no-check-certificate --no-parent --auth-no-challenge --no-cookies --save-headers --ignore-length --no-cache --ignore-case --no-dns-cache --random-wait --no-use-server-timestamps -A "$ACCEPT" -R "$REJECT" --continue -P $DIR $URL
}
#Infinate While Loop To Keep Menu Open At Bad Input
while :
do
########Menu#########
printf "Choose from the following operations:\n"
printf "[1] wget Method 1 (Simple)\n"
printf "[2] wget Method 2 (Exra Paramaters)\n"
printf "[3] spider with wget\n"
read -p "Your choice: " op
case $op in
1)
downloadSite
break
;;
2)
downlaod
break
;;
3)
spider
;;
*)
echo "invalid Input, Please Try Again."
esac
done
echo -e "\nThank You For Using My Simple Script, Please Feel Free To Modify/Share It!\n"
exit 1
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment