Shared SparkSession for tests

SparkSpec.scala

import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import org.my.tests.SparkSpec.sparkSession
import org.scalatest.flatspec.FixtureAnyFlatSpec
import org.scalatest.matchers.should.Matchers
import org.scalatest.Outcome



trait SparkSpec extends FixtureAnyFlatSpec with Matchers {

  type FixtureParam = SparkSession


  // see https://www.scalatest.org/user_guide/sharing_fixtures#withFixtureOneArgTest for an explanation
  def withFixture(test: OneArgTest): Outcome = {
    val theFixture = sparkSession

    try {
      withFixture(test.toNoArgTest(theFixture)) // "loan" the fixture to the test
    }
    finally sparkSession.sqlContext.clearCache() // clean up
  }

}

object SparkSpec {
  private lazy val sparkSession = SparkSession
    .builder()
    .config(
      new SparkConf()
        .setMaster("local[*]")
        .set("spark.sql.shuffle.partitions", "1")
    )

    .getOrCreate()
}

// Sample test
class MyTest1 extends SparkSpec {
  it should "work" in {sparkSession => // get a handle to the "loaned" spark session
    import sparkSession.implicits._

    List.empty[String].toDF.show
    succeed
  }
}