Tweet Harvest: Twitter User Data Dump-to-one-semi-clean-tweet-per-line file

tweet_harvest.rb

require 'json'

fo = File.open('/tmp/tweetstorm.txt','w')

Dir['./data/js/tweets/*.js'].each do |file|
  begin
    fh = File.open(file)
    data = fh.read
    # My Twitter dump had a first line that assigns the JSON data to a variable,
    # the rest was valid JSON. Example:
    # Grailbird.data.tweets_2009_07 =
    json_data = JSON.parse(data.split("\n")[1..-1].join)
    tweetz = json_data.map {|t| t['text'].gsub(/\@\w+/, '')}
    tweetz.each {|t| fo.puts(t) }
  ensure
    fh.close
  end
end
fo.close