Skip to content

Instantly share code, notes, and snippets.

@retroryan
Created May 29, 2015 15:39
Show Gist options
  • Save retroryan/00ee9c3e4f9925c684de to your computer and use it in GitHub Desktop.
Save retroryan/00ee9c3e4f9925c684de to your computer and use it in GitHub Desktop.
class DseSparkContextFactory extends SparkContextFactory {
import SparkJobUtils._
type C = SparkContext with ContextLike
def makeContext(config: Config, contextConfig: Config, contextName: String): C = {
val conf = configToSparkConf(config, contextConfig, contextName)
val sparkMaster = "dsetool sparkmaster".!!.trim
conf.set("spark.master",sparkMaster)
new SparkContext(conf) with ContextLike {
def sparkContext: SparkContext = this
def isValidJob(job: SparkJobBase): Boolean = job.isInstanceOf[SparkJob]
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment