Created
May 18, 2012 03:47
-
-
Save robvinson/2723040 to your computer and use it in GitHub Desktop.
Get index page from web servers
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/sh | |
DSTDIR=80open | |
mkdir $DSTDIR | |
is_empty() { | |
if [ -s $1 ]; then #exists and is not empty | |
return 1 | |
else | |
if [ -f $1 ]; then #file exists and is empty | |
return 0 | |
else | |
return 1 #doesn't exist | |
fi | |
fi | |
} | |
for i in `cat $1`; do | |
echo "[*] Getting $i" | |
#get the page by IP | |
wget -q -t 1 --timeout=10 --no-check-certificate "http://${i}" -O ${DSTDIR}/${i} | |
#get the page by fqdn (in case the web server uses vhosts) | |
fqdn=`dig -x $i +short` | |
fqdn_length=`expr length "$fqdn"` | |
if [ $fqdn_length -ne 0 ]; then #we have something for an fqdn | |
echo "[*] Getting $fqdn" | |
wget -q -t 1 --timeout=10 --no-check-certificate "http://${fqdn}" -O ${DSTDIR}/${i}-${fqdn} | |
# if the fqdn version and the IP version are identical we can get rid of one of them | |
diff -q ${DSTDIR}/${i} ${DSTDIR}/${i}-${fqdn} > /dev/null | |
if [ $? -eq 0 ]; then | |
echo "[-] FQDN and IP versions are the same, removing ${i}-$fqdn" | |
rm ${DSTDIR}/${i}-${fqdn} | |
fi | |
fi | |
#remove empty files | |
is_empty ${DSTDIR}/${i} | |
if [ $? -eq 0 ]; then | |
echo "[-] Removing empty file ${DSTDIR}/${i}" | |
rm ${DSTDIR}/${i} | |
fi | |
is_empty ${DSTDIR}/${i}-${fqdn} | |
if [ $? -eq 0 ]; then | |
echo "[-] Removing empty file ${DSTDIR}/${i}-${fqdn}" | |
rm ${DSTDIR}/${i}-${fqdn} | |
fi | |
done |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment