twitterchive.rb

#!/usr/bin/env ruby

tag = "ASD"

### Don't need to edit below here
%w{ rubygems hpricot open-uri }.each {|gem| require gem}
next_url = "http://search.twitter.com/search.atom?tag=#{tag}&lang=all&rpp=50"

fields = ["id","title", "published","author/name","author/uri"]
entries = [fields.join(",") + ",link"]
while (next_url)
  begin
    puts next_url
    doc = Hpricot.parse(open(next_url))

    entries << (doc/:entry).collect do |entry|
      fields.collect {|f| (entry/f).inner_text }.join(",") + ",#{(entry/"link").first["href"]}"
    end
    next_url = (doc/"link[@rel=next]").first["href"]
    sleep(0.2)
  rescue
    next_url = nil
  end
end


File.open("#{tag}.csv","w") do |file|
  file << entries.join("\n")
end