code partyで書いたtf-idfのRuby実装

tfidf.rb

text = File.read "./utamap.test.txt"

vocabraries = []
word2id = {}

text.split("\n").each do |doc|
  words = doc.split(" ")
  words.each do |word|
    unless word2id.has_key?(word)
      vocabraries << word
      word2id[word] = vocabraries.count - 1
    end
  end
end

hist_all = []

hist_all = text.split("\n").map do |doc|
  words = doc.split(" ")
  hist = {}
  words.each do |word|
    if hist.has_key?(word)
      hist[word] += 1
    else
      hist[word] = 1
    end
  end
  hist
end

idf = hist_all.reduce({}) do |df, hist|
  hist.keys.each do |word|
    if df.has_key?(word)
      df[word] += 1
    else
      df[word] = 1
    end
  end
  df
end.map do |word, count|
  [word, Math.log10(hist_all.count.to_f / count)]
end.to_h


docs = text.split("\n")
hist_all.each_with_index do |hist, index|
  puts "======================================="
  puts docs[index]
  total = hist.values.sum
  hist.each do |word, count|
    tf = count.to_f / total
    puts "#{word} => tfidf: #{tf * idf[word]}"
  end
end