A crawler used to grab all the scripts of big bang

crawler_big_bang_scripts.rb

require 'rubygems'
require 'mechanize'

agent = Mechanize.new

page = agent.get('http://bigbangtrans.wordpress.com/')

page.links.select {|l| l.text =~ /^Series/}.each do |link|
  puts link.text
  sub_page = link.click
  title = sub_page.search("h2.title").first.content
  scripts = []

  sub_page.search("div.entrytext > p").each do |i|
    scripts << i.content
  end

  File.open("#{title.gsub(/\W+/, '_')}.txt", "w") do |f|
    f.write(scripts * "\n\n")
  end
end