consumer streaming tweets and stuff them into indextank

twitter_fetcher.rb

class TwitterFetcher 

  def perform

    api  = IndexTank::Client.new(ENV['INDEXTANK_API_URL'] || index_tank_url)
    index = api.indexes 'idx'

    streaming_url = 'http://stream.twitter.com/1/statuses/sample.json'
       EventMachine.run do
        http = EventMachine::HttpRequest.new(streaming_url).get :head => { 'Authorization' => [ TWITTER_USERNAME, TWITTER_PASSWORD ] }
        buffer = ""
        http.stream do |chunk|
          buffer += chunk
            while line = buffer.slice!(/.+\r?\n/)
              tweet = JSON.parse(line)
              if tweet['text']
               index.document(tweet["id"].to_s).add({:text => tweet["text"]})
              end
            end
         end
      end
   end
end