Get index page from web servers

dl.sh

#!/bin/sh

DSTDIR=80open
mkdir $DSTDIR

is_empty() {
  if [ -s $1 ]; then #exists and is not empty
    return 1
  else
    if [ -f $1 ]; then #file exists and is empty
      return 0
    else
      return 1 #doesn't exist
    fi  
  fi  
}

for i in `cat $1`; do
  echo "[*] Getting $i"
  #get the page by IP
  wget -q -t 1 --timeout=10 --no-check-certificate "http://${i}" -O ${DSTDIR}/${i}

  #get the page by fqdn (in case the web server uses vhosts)
  fqdn=`dig -x $i +short`
  fqdn_length=`expr length "$fqdn"`
  if [ $fqdn_length -ne 0 ]; then #we have something for an fqdn
    echo "[*] Getting $fqdn"
    wget -q -t 1 --timeout=10 --no-check-certificate "http://${fqdn}" -O ${DSTDIR}/${i}-${fqdn}
    # if the fqdn version and the IP version are identical we can get rid of one of them
    diff -q ${DSTDIR}/${i} ${DSTDIR}/${i}-${fqdn} > /dev/null
    if [ $? -eq 0 ]; then
      echo "[-] FQDN and IP versions are the same, removing ${i}-$fqdn"
      rm ${DSTDIR}/${i}-${fqdn}
    fi  
  fi  
  
  #remove empty files
  is_empty ${DSTDIR}/${i}
  if [ $? -eq 0 ]; then
    echo "[-] Removing empty file ${DSTDIR}/${i}"
    rm ${DSTDIR}/${i}
  fi  
  is_empty ${DSTDIR}/${i}-${fqdn}
  if [ $? -eq 0 ]; then
    echo "[-] Removing empty file ${DSTDIR}/${i}-${fqdn}"
    rm ${DSTDIR}/${i}-${fqdn}
  fi  

  
done