groonga-token-counter.rb

require "groonga"

database_path = ARGV[0]
table_name = ARGV[1]
index_column_name = ARGV[2]

Groonga::Database.open(database_path)
index = Groonga["#{table_name}.#{index_column_name}"]
terms = Groonga::Hash.create(:key_type => Groonga::Type::SHORT_TEXT,
                             :value_type => Groonga::Type::UINT32)

index.table.open_cursor do |table_cursor|
  index.open_cursor(table_cursor, :with_position => false) do |cursor|
    cursor.each do |posting|
      if !posting.term.key.ascii_only? && posting.term.key.size == 2
        terms.add(posting.term.key)
        value = terms.value(posting.term.key) + posting.term_frequency
        terms.set_value(posting.term.key, value)
      end
    end
  end
end

sorted = terms.sort([{:key => "_value",:order => "descending"}])
sorted.each do |record|
  puts "#{record._key},#{record._value}"
end

# Ruby hash
# terms = Hash.new(0)
# terms[posting.term.key] += posting.term_frequency
# terms.sort_by{|key,value| -value}.each do |key, value|
#  puts "#{key},#{value}"
# end