Code for TFIDF-based algorithm for obtaining relevant research topics for Rewired State GTR hackathong

removal1.rb

filter = Stopwords::Snowball::Filter.new "en"
unique_words = filter.filter(unique_words).uniq

sanitise1.rb

unique_words.reject! { |word, tfidf| query.split.map(&:sanitize).include? word }

sanitise2.rb

word_corpus = File.read("words.txt").split("\n").map(&:downcase)
unique_words.reject! { |word, tfidf| word_corpus.include? word }

split1.rb

String.class_eval do
  # Remove all non alphanumeric characters
  def alphanumeric_only
    return self.gsub(/[^0-9a-z ]/i, '')
  end

  # Sanitize a string  by removing all non alphanumeric characters,
  # downcasing it and removing any excess spaces
  def sanitize
    return self.alphanumeric_only.downcase.squeeze " "
  end
end

split2.rb

words = titles.map { |title| title.split(" ") }.flatten(1).map(&:sanitize)
unique_words = words.uniq

tfidf1.rb

unique_words = unique_words
                .group_by { |w| w }
                .map { |w, ws| [w, ws.count] }
                .select { |w, count| count > 10 }

tfidf2.rb

max_freq = unique_words.map { |word, count| count }.max
unique_words.map! { |word, count| [word, tf(count, maxf) * idf(word, documents)] }
unique_words.sort_by! { |word, tfidf| tfidf }
unique_words.reverse!

tfidf3.rb

def tf freq, max_freq
  0.5 + 0.5*freq/max_freq
end

tfidf4.rb

def idf(word, documents)
  cardinality = documents.count
  num_docs = documents.select { |doc| doc.include? word }.count
  Math.log10(cardinality * 1.0 / (1 + num_docs))
end