spider.rb

require 'rubygems'
require 'spider' # sudo gem install spider

Spider.start_at('http://localhost/') do |s|  
  # Limit the pages to just this domain.  
  s.add_url_check do |a_url|  
    a_url =~ %r{^http://localhost.*}  
  end  
  
  # Handle 404s.  
  s.on 404 do |a_url, err_code|  
    puts "URL not found: #{a_url}"  
  end  
  
  # Handle 500s.
  s.on 500 do |a_url, err_code|  
    puts "URL 500 error: #{a_url}"  
  end
  
  # Handle 2xx.  
  s.on :success do |a_url, code, headers, body|  
    puts "URL OK: #{a_url}"  
  end  
  
  # Handle everything.  
  s.on :any do |a_url, resp|  
    puts "URL returned anything: #{a_url} with this code #{resp.code}"  
  end  
end