Download all tweets from the twitter search API for a given search term (limited to their maximum of 1500)

scrape_tweets.rb

#! /usr/bin/env ruby
require "fileutils"
search_term = ARGV[0]
if search_term
  time = Time.now
  directory_path = File.dirname(__FILE__) + "/tweets/" + search_term + "_" + time.to_i.to_s
  FileUtils.mkdir_p(directory_path)
  directory = Dir.new(directory_path)
  (1..15).each do |i|
    `curl "http://search.twitter.com/search.json?q=#{search_term}&rpp=100&page=#{i}&include_entities=true&result_type=mixed" > #{directory.path}/#{i}.json`
  end
  puts "Scraped to #{directory_path}"
else
  puts "./scrape_tweets.rb <your search term in quotes>"
end