detect languages and tokenize via core foundation

macruby_string_token_lang_detect.rb

framework 'Foundation'
class String
  def language
    CFStringTokenizerCopyBestStringLanguage(self, CFRangeMake(0, self.size))
  end
  
  def tokens
      str_array = []
      stok = CFStringTokenizerCreate(nil,self,[0,self.length],0,nil)
      CFStringTokenizerGetCurrentTokenRange(stok)
      has_next = CFStringTokenizerAdvanceToNextToken(stok)
      until has_next == 0 do
          range = CFStringTokenizerGetCurrentTokenRange(stok)
          str_array << self[range.location,range.length]
          has_next = CFStringTokenizerAdvanceToNextToken(stok)
      end
      str_array
  end
end


puts "Bonne année!".language
# => "fr"
puts "Happy new year!".language
# => "en"
puts "¡Feliz año nuevo!".language
# => "es"
puts "Felice anno nuovo!".language
# => "it"
puts "أعياد سعيدة".language
# => "ar"
puts "@umran_chaelle ne oluo ümran nedir olay hattı kırmalar falan".language
puts "明けましておめでとうございます。".language
puts "明けましておめでとうございます。".tokens
puts "".language
puts "".tokens