Skip to content

Instantly share code, notes, and snippets.



Last active Dec 13, 2016
What would you like to do?
SCarman's spark job API on top of existing API
package spark.jobserver.api
import com.typesafe.config.Config
import org.scalactic._
import spark.jobserver.api._
trait ContextProvider[C] {
val ctx: C with ContextLike = null
def context: C with ContextLike = ctx
trait SimpleSparkJob[R, CX] {
type JobOutput = R
type JobData = Config
type C = CX
def runJob(runtime: JobEnvironment, data: Config): R
def validate(runtime: JobEnvironment, config: Config): Config
val baseSparkJob = new SparkJobBase with ContextProvider[CX] {
def runJob(sc: C, runtime: JobEnvironment, data: JobData): JobOutput = {
ctx = sc
runJob(runtime, data)
def validate(sc: C, runtime: JobEnvironment, config: Config): JobData Or Every[ValidationProblem] = {
try {
Good(validate(runtime, config))
} catch {
case e: Exception => Bad(SingleProblem(e.getMessage))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.