Twitter のストリームを取得して Mroonga な DB に突っ込みまくるサンプル

Twitter Stream Sampling to Mroonga

#!/usr/bin/env ruby
# coding: utf-8

require 'net/https'
require 'uri'
require 'rubygems'
require 'json'
require 'pp'
require 'data_mapper'

print "account: "
account = $stdin.gets.chomp
print "password: "
system "stty -echo"
pass = $stdin.gets.chomp
system "stty echo"
puts

class Tweets
  include DataMapper::Resource
  property :id, Serial
  property :tweet, Text
  property :tweeted_at, DateTime
  property :name, String
  property :screen_name, String
  property :profile_image_url, String
  property :language, String
  property :raw, Text
end
DataMapper.finalize

DataMapper.setup(
  :default, {
    :adapter => "mysql",
    :database => "testmroonga",
    :username => "root",
    :password => "",
    :socket => "/var/lib/mysql/mysql.sock"
  }
)

uri = URI.parse('https://stream.twitter.com/1.1/statuses/sample.json')
https = Net::HTTP.new(uri.host, uri.port)
https.use_ssl = true
https.ca_file = '/etc/pki/tls/cert.pem'
https.verify_mode = OpenSSL::SSL::VERIFY_PEER
https.verify_depth = 5
https.start do
  request = Net::HTTP::Get.new(uri.request_uri)
  # Streaming APIはBasic認証のみ
  request.basic_auth(account, pass)
  https.request(request) do |response|
    raise 'Response is not chuncked' unless response.chunked?
    response.read_body do |chunk|
      begin
        # 空行は無視する = JSON形式でのパースに失敗したら次へ
        status = JSON.parse(chunk) rescue next
        # 空行は無視する = JSON形式でのパースに失敗したら次へ
        # 削除通知など、'text'パラメータを含まないものは無視して次へ
        next unless status['text']
        #next unless status['user']['lang'] == "ja"
        user = status['user']
        puts "#{user['screen_name']}: #{status['text']}"
        #pp status
        tweets = Tweets.new({
          :tweet => status['text'],
          :tweeted_at => status['created_at'],
          :name => status['user']['name'],
          :screen_name => status['user']['screen_name'],
          :profile_image_url => status['user']['profile_image_url'],
          :language => status['user']['lang'],
          :raw => JSON.generate(status)
        })
        tweets.save!
      rescue
      end
    end
  end
end