bjornjohansen

#!/usr/bin/php
<?php
/**
 * @license http://www.wtfpl.net/txt/copying/ WTFPL
 */

date_default_timezone_set( 'UTC' );


$sitemaps = array(

suzannealdrich

wget --spider -o wget.log -e robots=off -r -l 5 -p -S --header="X-Bypass-Cache: 1" --limit-rate=124k www.example.com

# Options explained
# --spider: Crawl the site
# -o wget.log: Keep the log
# -e robots=off: Ignore robots.txt
# -r: specify recursive download
# -l 5: Depth to search. I.e 1 means 'crawl the homepages'.  2 means 'crawl the homepage and all pages it links to'...
# -p: get all images, etc. needed to display HTML page
# -S: print server response