Skip to content

Instantly share code, notes, and snippets.

@SingularBunny
Created November 20, 2019 18:24
Show Gist options
  • Save SingularBunny/ec410c40ac41d82bd448ad13024c9db7 to your computer and use it in GitHub Desktop.
Save SingularBunny/ec410c40ac41d82bd448ad13024c9db7 to your computer and use it in GitHub Desktop.
class ExecutionListenerManagerSuite extends SparkFunSuite with LocalSparkSession {
import CountingQueryExecutionListener._
test("register query execution listeners using configuration") {
val conf = new SparkConf(false)
.set(QUERY_EXECUTION_LISTENERS, Seq(classOf[CountingQueryExecutionListener].getName()))
spark = SparkSession.builder().master("local").appName("test").config(conf).getOrCreate()
spark.sql("select 1").collect()
spark.sparkContext.listenerBus.waitUntilEmpty()
assert(INSTANCE_COUNT.get() === 1)
assert(CALLBACK_COUNT.get() === 1)
val cloned = spark.cloneSession()
cloned.sql("select 1").collect()
spark.sparkContext.listenerBus.waitUntilEmpty()
assert(INSTANCE_COUNT.get() === 1)
assert(CALLBACK_COUNT.get() === 2)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment