Scored word generation

scored_word_generation

  val PIPELINE = new AnnotationPipeline(
      new PtbRegexTokenizer, new StanfordMaxentTagger, new StanfordNameTagger, new HyphenWordExtractorAnnotator,
      new MweExtractorAnnotator(GLOSSARY_WORDS), new SimplePatternAnnotator
  )


  def toScoredWords(md : Metadata , as : AnnotatedSentence) : java.util.List[ScoredWord] = {
    val annoSentence = toAnnoSentence(as)
    annoSentence.setMetadata(md)
    PIPELINE.process(annoSentence)

    val scoredWords = new ArrayList[ScoredWord]
    ScoredWordExtractor.getScoredWords(annoSentence, List("tokens", "hyphenword", "ne", "mwe"), Some(GLOSSARY_WORDS), true, new ScoredWordRanker).foreach(swa => {
      val sw = toScoredWord(swa)
      sw.setSentenceId(as.getId)
      scoredWords.add(sw)
    })
    scoredWords
  }