ExampleSharedDF.scala

import org.apache.spark.SparkContext
import org.apache.spark.sql.hive.HiveContext
import com.typesafe.config.Config
import spark.jobserver.NamedRddSupport
import spark.jobserver.SparkHiveJob
import spark.jobserver.SparkJob
import spark.jobserver.SparkJobValid
import spark.jobserver.SparkJobValidation

/**
 * Example Spark Job using spark-jobserver
 * Shares a RDD / DataFrame using {@link SparkHiveJob} leaving it cached in memory.
 *
 * @author John Muller
 */
object ExampleNamedDF extends SparkHiveJob {

  // This is the method that gets called by spark-jobserver when it submits a SparkJob
  override def runJob(sqlContext: HiveContext, jobConfig: Config): Any = {
    // Select some stuff from Hive
    val merckDataFrame = sqlContext.sql("SELECT a, b, c FROM TableA")
    // Leave the dataframe cached in memory.
    sqlContext.cacheTable("CACHED_DATA")
    // The next SparkHiveJob will be able to access the in-memory DataFrame with the below:
    // sqlContext.sql("SELECT * FROM CACHED_DATA")
  }

  // Dummy validation for this job, will always return that the job is valid.
  override def validate(sqlContext: HiveContext, config: Config): SparkJobValidation = SparkJobValid
}