bash function that crawls for non-web URL paths

crawl.sh

function crawl_for_urls() {
    if [ $# != 1 ]; then
        echo "usage: crawl_for_urls URL" >2
        exit 1
    fi  

    wget --spider --force-html -r "$1" 2>&1 | \ 
        grep '^--' | \ 
        awk '{ print \$3 }' | \ 
        grep -v '\.\(css\|js\|png\|gif\|jpg\)$'
}

test.sh

crawl_for_urls "http://www.infoq.com"