check a bunch of pages

check404s.sh

#!/bin/bash

NOWD=$(date +"%Y-%m-%d")

URLS="
http://www.domain.com/page1.html
http://www.domain.com/page2.html
http://www.domain.com/pagen.html
http://www.domain.com/and/so/on.html
"

clean_var()
{
INVAR=$1
VARLEN=${#INVAR}
VAR=$(echo -ne "$RESPONSE" | grep -m 1 "$INVAR")
VAR=${VAR:VARLEN}
VAR=${VAR/^M/}
VAR=${VAR/ /}
VAR=${VAR:="Nan"}
echo -n "$VAR"
}

for URL in ${URLS}
do
        NOWT=$(date +"%T")
        RESPONSE=$(curl -j -D - -o /dev/null -L -A "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; TigerTeam; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3" --header "X-Forwarded-For: 123.123.123.123" --connect-timeout 90 -m 120 --silent --progress-bar --write-out 'STATS%{time_namelookup},%{time_connect},%{time_starttransfer},%{time_total},%{http_code},%{size_header},%{size_download},%{speed_download},%{url_effective}' ${URL})
        # extract some specific X header info... ;-)
        XHOST=`clean_var X-Host:`
        XVERSION=`clean_var X-Version:`
        
        STATS=$(echo "$RESPONSE" | grep "STATS")
        STATS=${STATS:5}

        printf "$NOWD,$NOWT,$XHOST,$XVERSION,$STATS,$URL\n"  >> ~/$NOWD.dat
done