Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
class DseSparkContextFactory extends SparkContextFactory {
import SparkJobUtils._
type C = SparkContext with ContextLike
def makeContext(config: Config, contextConfig: Config, contextName: String): C = {
val conf = configToSparkConf(config, contextConfig, contextName)
val sparkMaster = "dsetool sparkmaster".!!.trim
conf.set("spark.master",sparkMaster)
new SparkContext(conf) with ContextLike {
def sparkContext: SparkContext = this
def isValidJob(job: SparkJobBase): Boolean = job.isInstanceOf[SparkJob]
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment