Screen Scraping Script for Kickstarter Projects

kickstarter_csv.rb

# Usage: ruby kickstarter_csv.rb output.csv

require 'rubygems'
require 'kickstarter'

LAST_REPORT_DATE = Date.parse("April 28, 2011")
FIELDS = [:name, :pledge_amount, :pledge_percent, :pledge_deadline, :owner]


def fetch_projects_for_category category, file
   page = 1
   finished = false
   while !finished
       projects = Kickstarter.by_category(category, :page => page, :type => :successful)
       valid_projects = projects.reject { |p| p.pledge_deadline < LAST_REPORT_DATE }
       valid_projects.each do |project|
           file.write category.to_s + ","
           file.write FIELDS.map { |attr_name| project.send(attr_name).to_s.gsub(/[,"]/,"") }.join(",") + "\n"
       end
       page += 1
       finished = projects.size > valid_projects.size
   end
end


File.open(ARGV[0], 'w') do |file|
    file.write "category, "
    file.write FIELDS.join(", ") + "\n"
    Kickstarter::Categories.each_pair do |key, value|
       fetch_projects_for_category key, file
    end
end