markov chain table

make_trans_prob.rb

# -*- coding: utf-8 -*-
require 'natto'
require 'msgpack'
require 'mongo'

texts = ['shirobako★が3巻が見たい', '大久保瑠美さんに会いたい']

trans_freq = Hash.new{|h, k| h[k] = Hash.new{0.0}}

texts.each do |txt|
  natto = Natto::MeCab.new
  ary = ['***BOS***']
  natto.parse(txt) do |n|
    w = n.surface
    feature = n.feature.split(',') 
    type_1 = feature[0]
    type_2 = feature[1]
    next if type_1 == '記号'
    next if type_2 == '数'
    p w
    ary.push w
  end

  ary.each_cons(2) do |elem|
    f, t = elem
    trans_freq[f][t] += 1
  end
end

# convert transition probability
trans_prob = Hash.new{|h, k| h[k] = Hash.new{0.0}}
trans_freq.each_pair do |f, h|
  sum = h.values.inject(:+)
  h.each_pair do |t, v|
    trans_prob[f][t] = v / sum
  end
end

# save trans_prob
open('./anime_title_prob.dump', 'wb'){|f|
  f.puts trans_prob.to_msgpack
}

# load trans_prob
t = { }
open('./anime_title_prob.dump', 'rb'){|f|
  t = MessagePack.unpack(f.read.chomp)
}

p t