QuoraQuestionsPairsPipeline.scala

val tokenizer: Param[RegexTokenizer] =
    new Param(this, "tokenizer", "Breaks the sentences into individual words.")
setDefault(tokenizer, new RegexTokenizer().setPattern("""[\p{Punct} ]"""))
val stopwordsRemover: Param[StopWordsRemover] =
  new Param(this, "stopwords", "Drops stopwords from input text.")
  // calling code needs to provide the list of stopwords, default is empty
setDefault(stopwordsRemover, new StopWordsRemover())

// A convenience function to refer to columns in our pipeline
private val questionsCols = Array("question1", "question2")
private def questions(suffix: String) = questionsCols.map(_ + suffix)
 
private def tokenizePipeline(): Array[PipelineStage] = {
  val mcTokenizer = new MultiColumnPipeline(
    .setStage($(tokenizer))
    .setInputCols(questions(""))
    .setOutputCols(questions("all_tokens"))
  val mcStopwordsRemover = new MultiColumnPipeline()
    .setStage($(stopwordsRemover))
    .setInputCols(mcTokenizer.getOutputCols)
    .setOutputCols(questions("tokens"))
  Array(mcTokenizer, mcStopwordsRemover)
}