railsでcrawler、botのアクセスを制御する ref: http://qiita.com/toyama0919/items/52ffaeb6142c3770ed7a

is_crawler.rb

# -*- coding: utf-8 -*-
def crawler? ua
  crawler_ua = [
    "crawler",
    "bot",
    "spider",
    "YahooFeedSeeker",
    "Y!J-(?:BSC|BRO|SRD)/",
    "internal +dummy +connection",
    "libwww",
    "HyperEstraier",
    "curl",
    "wget",
    "facebookexternalhit",
    "ia_archiver",
    "Tumblr"
  ]

  crawler_ua.each{ |reg|
    return true if (/#{reg}/i =~ ua) != nil
  }
  return false
end

p crawler?(request.user_agent)
p crawler?("Y!J-BSC/YATS crawler")
p crawler?('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.56 Safari/536.5')