crawler_algorithm.rb

gistfile1.txt

                        while(list of unvisited URLs is not empty) {
    take URL from list
    fetch content
    record whatever it is you want to about the content
    if content is HTML {
        parse out URLs from links
        foreach URL {
           if it matches your rules
              and it's not already in either the visited or unvisited list
              add it to the unvisited list
        }
    }
}