crawler.rb

#!/usr/local/bin/ruby

# crawler.rb
# by: Jason Larsen
# a generic web crawler that allows the user to do whatever they want by passing blocks
# @version 0.7
# 14 Dec 2009
# 0.6 things seem to be working well
# 0.7 modified so that URL's being added to the queue truncate fragments,
# this should save a lot of work

require 'rubygems'
require 'net/http'
require 'nokogiri'
require 'open-uri'
require 'fileutils'

class Crawler
    attr_accessor :verbose, :mode
    attr_reader :visited, :errors, :base_url, :time
    
    def initialize url
        @base_url = URI::parse url
        @base_url.normalize!
        @visited, @errors, @queue, @pages = [], {}, [@base_url], 0
    end
    
    def crawl
        start_time = Time::now
        until @queue.empty? do
			# FIXME remove when fixed weird error
			puts
			puts "Queue Before:"
            puts @queue.size < 4 ? @queue : @queue.size
            puts "Visited size: #{@visited.size}"
			
            url = @queue.shift
            puts "Crawling #{url}" if verbose
			
            begin
                src = Nokogiri::HTML::parse(open url.to_s)
            rescue Exception => e
                puts e if verbose
                if errors[e.to_s] then
                   errors[e.to_s] += 1
                else
                    errors[e.to_s] = 0
                end
				next # try the next url
            end
            
            for link in src.search('a')
                begin
                    href = URI::parse link['href']
                    href.normalize!
                    href = url + href if href.relative?
                rescue Exception
                    next
                end
                
                # skip any non-http links e.g mailto, etc.
                next if not href.scheme == 'http'
                # add it to the queue if we need to
                enqueue href
				
				if scope? href then
					if href.path.end_with? "/" then
						link['href'] += "index.html"
					else
						path = href.path[0.. href.path.rindex('/')]
						path = href.path + "index.html" if href.path.end_with? "/"
						link['href'] = File.basename(href.path).gsub(/[.].*/,'.html')
					end
				end
				
            end
            
            just_visited url
                        
            # write to file
			# figure out where we're writing it
			url.path = url.path + "index.html" if url.path.end_with? '/'
			file_name = File.basename(url.path).gsub(/[.].*/,'') + '.html'
			dirs = File.dirname(url.host + url.path)
			# make the directories
			FileUtils.makedirs(dirs) unless File.directory?(dirs)
			file = File.open(dirs + '/' + file_name, 'w')
			file.puts src.inner_html 
			file.close
			puts "Saved as #{file.path}"
            
            # FIXME remove when fixed weird error
			puts "Queue After:"
            puts @queue.size < 4 ? @queue : @queue.size
            puts "Visited size: #{@visited.size}"
        end
        @time = Time::now - start_time
    end
    
    def report
        error_report = ""
        @errors.each { |error,count| error_report += "\t#{error}: #{count}\n" }
        "Pages cached: #{@pages}\nTime elapsed: #{@time} seconds\nErrors:\n" + error_report
    end
    
    private 
    
    def enqueue url
        url.fragment = nil # by getting rid of the fragments we save a lot of work
        if scope? url and not visited? url and not queued? url then
			if @queue.size < 4 then
				puts "#{url} is in scope" if scope? url
				puts "#{url} is not visited" if not visited? url
				puts "#{url} is not queued" if not queued? url
			end
			puts "...Queuing #{url}..." if verbose
			@queue.push url
        end
    end
    
    def just_visited url
        @pages += 1
        @visited.push url
		puts "visited #{url}" if @queue.size < 4 and visited? url
    end
	
	def scope? url
        return false if not url.scheme == @base_url.scheme and not url.host == @base_url.host
        url.path.start_with? @base_url.path[0..@base_url.path.rindex('/')]
    end
	
	def visited? url
		@visited.include? url
	end
	
	def queued? url
		@queue.include? url
	end
end

crawler = Crawler.new 'http://saas.byu.edu/catalog/2005-2006ucat/'
crawler.verbose = true
crawler.crawl
puts crawler.report