SubmissionWriter.scala

def transform(df: Dataset[_]): DataFrame = {
 import df.sparkSession.implicits.newProductEncoder
  val predictions = $(model).transform(df).select("p", "id").as[(DenseVector, String)]
  predictions map { case (p, testId) =>
    (p.values.last, testId)
  } toDF("is_duplicate", "test_id")
}

def writeSubmissionFile(features: Dataset[Features], submissionFilePath: String): Unit = {
  val csvOptions = Map("header" -> "true", "escape" -> "\"")
  val submission = transform(features)
  submission.repartition(1).write.options(csvOptions).csv(submissionFilePath)
}