QuoraQuestionsPairsPipeline.scala

val countVectorizer: Param[CountVectorizer] =
  new Param(this, "countVectorizer", "Converts words into numerical ids.")
setDefault(countVectorizer, new CountVectorizer)
val idf: Param[IDF] =
  new Param(this, "idf", "Calculate weights for vector representation of tokens.")
setDefault(idf, new IDF())

private def vectorizePipeline(): Array[PipelineStage] = {
  val mcTf = new MultiColumnPipeline()
    .setInputCols(questions("tokens"))
    .setOutputCols(questions("tf"))
    .setStage($(countVectorizer))
  val mcIdf = new MultiColumnPipeline()
    .setInputCols(questions("tf"))
    .setOutputCols(questions("tfidf")
    .setStage($(idf))
  Array(mcTf, mcIdf)
}