get_all_words_in.cpp

std::vector<Word> get_all_words_in(const std::vector<Page>& pages) {
  std::vector<Word> words;
  std::map<std::string, int> word_conv;
  int num_words = 0;
  for(const Page& page : pages) {
    std::vector<std::string> raw_words = filter_with_regex(get_words_in(page.url()), ".*\\.html$", false);
    for(int i = 0; i < static_cast<int>(raw_words.size()); ++i) {
      if(!word_conv.count(raw_words[i])) {
	word_conv[raw_words[i]] = num_words++;
	words.emplace_back(word_conv[raw_words[i]], raw_words[i]);
      }
      words[word_conv[raw_words[i]]].is_located_on(page.id(), i);
    }
  }
  return words;
}