Skip to content

Instantly share code, notes, and snippets.

@velvia

velvia/sparkjobapi.scala

Last active Dec 13, 2016
Embed
What would you like to do?
SCarman's spark job API on top of existing API
package spark.jobserver.api
import com.typesafe.config.Config
import org.scalactic._
import spark.jobserver.api._
trait ContextProvider[C] {
val ctx: C with ContextLike = null
def context: C with ContextLike = ctx
}
trait SimpleSparkJob[R, CX] {
type JobOutput = R
type JobData = Config
type C = CX
def runJob(runtime: JobEnvironment, data: Config): R
def validate(runtime: JobEnvironment, config: Config): Config
val baseSparkJob = new SparkJobBase with ContextProvider[CX] {
def runJob(sc: C, runtime: JobEnvironment, data: JobData): JobOutput = {
ctx = sc
runJob(runtime, data)
}
def validate(sc: C, runtime: JobEnvironment, config: Config): JobData Or Every[ValidationProblem] = {
try {
Good(validate(runtime, config))
} catch {
case e: Exception => Bad(SingleProblem(e.getMessage))
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.