Looks for the existence of dirs on a numbered list. Done in order to learn net/http and threads.

crawler.rb

#!/usr/bin/env ruby
#
###################################################
#                crawler.rb                       #
# Pedro Vanzella - pedro@pedrovanzella.com        #
#                                                 #
# Looks for the existence of directories on a     #
# numbered, but sparse, list                      #
###################################################

require "net/http"
require "uri"

def crawl(a,b)
  (a..b).each do |n|
    uri = URI.parse("http://twitpic.com/#{n}/")
# I know twitpic has alphanumeric URLs, but this was not the original URL anyway ;)

    http = Net::HTTP.new(uri.host, uri.port)
    request = Net::HTTP::Get.new(uri.request_uri)

    begin
      response = http.request(request)
      puts "[#{n}] => [#{response.code}]" unless response.code == '404'
    rescue Timeout::Error
      puts "Timeout on [#{n}]. Perhaps we're being throtled?"
    end

  end
end

if ARGV[2] == nil
  puts "USAGE: ruby crawler.rb FIRST LAST THREADS"
  exit
end

x = ARGV[0].to_i
y = ARGV[1].to_i
threads = ARGV[2].to_i

t = [] # Threads array
threads.times do |n|
  # Watch for division by zero!
  first = x + (n)*(y / threads) 
  last = x + (n + 1)*(y / threads) - 1
  # I've got an off-by-some on uneven divisions here, somewhere.

  puts "[#{n}] (#{first}..#{last})"
  t[n] = Thread.new { crawl(first, last) } 
end

# Join our threads
t.each do |n|
  n.join
end