Parallel processing with Scala-Spark

ParallelProcessing.scala

object ParallelProcessing {

  val queries: List[(String, String)] = List(
    ("SELECT * FROM ABC", "output1"), 
    ("SELECT * FROM XYZ", "output2")
  )

  // Just use parallel collection instead of futures, that's it
  queries.par foreach { 
    case (query, path) =>
      val dataPath = s"${pathPrefix}/{path}"
      executeAndSave(query, dataPath)
  }
  
  def executeAndSave(query: String, dataPath: String)(implicit context: Context): Unit = {
    println(s"$query starts")
    context.spark.sql(query).write.mode("overwrite").parquet(dataPath)
    println(s"$query completes")
  }
  
}

Did you mean SparkContext??

Oh wow ❤️