QuoraQuestionsPairsPipeline.scala

val lda: Param[LDA] =
  new Param(this, "lda", "Convert each question into a weighted topic vector.")
setDefault(lda, new LDA())
private def ldaPipeline(): Array[PipelineStage] = {
  // The "em" optimizer is distributed, supports serialization, but is disk hungry and slow.
  // The "online" runs in the driver, is fast but cannot be serialized.
  // We use the latter, since this model is only used to create a submission and nothing else.
  val optimizer = "online"
  val ldaEstimator = $(lda)
   .setOptimizer(optimizer)
   .setFeaturesCol("tmpinput").setTopicDistributionCol("tmpoutput")
  val mcLda = new MultiColumnPipeline()
   .setInputCols(questions("tfidf"))
   .setOutputCols(questions("lda"))
   .setStage(ldaEstimator, ldaEstimator.getFeaturesCol, ldaEstimator.getTopicDistributionCol)
  Array(mcLda)
}
view raw