Create a gist now

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Get index page from web servers
#!/bin/sh
DSTDIR=80open
mkdir $DSTDIR
is_empty() {
if [ -s $1 ]; then #exists and is not empty
return 1
else
if [ -f $1 ]; then #file exists and is empty
return 0
else
return 1 #doesn't exist
fi
fi
}
for i in `cat $1`; do
echo "[*] Getting $i"
#get the page by IP
wget -q -t 1 --timeout=10 --no-check-certificate "http://${i}" -O ${DSTDIR}/${i}
#get the page by fqdn (in case the web server uses vhosts)
fqdn=`dig -x $i +short`
fqdn_length=`expr length "$fqdn"`
if [ $fqdn_length -ne 0 ]; then #we have something for an fqdn
echo "[*] Getting $fqdn"
wget -q -t 1 --timeout=10 --no-check-certificate "http://${fqdn}" -O ${DSTDIR}/${i}-${fqdn}
# if the fqdn version and the IP version are identical we can get rid of one of them
diff -q ${DSTDIR}/${i} ${DSTDIR}/${i}-${fqdn} > /dev/null
if [ $? -eq 0 ]; then
echo "[-] FQDN and IP versions are the same, removing ${i}-$fqdn"
rm ${DSTDIR}/${i}-${fqdn}
fi
fi
#remove empty files
is_empty ${DSTDIR}/${i}
if [ $? -eq 0 ]; then
echo "[-] Removing empty file ${DSTDIR}/${i}"
rm ${DSTDIR}/${i}
fi
is_empty ${DSTDIR}/${i}-${fqdn}
if [ $? -eq 0 ]; then
echo "[-] Removing empty file ${DSTDIR}/${i}-${fqdn}"
rm ${DSTDIR}/${i}-${fqdn}
fi
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment