Skip to content

Instantly share code, notes, and snippets.

import org.apache.crunch.Source
import org.apache.crunch.impl.mr.run.CrunchInputFormat
import org.apache.avro.mapred.AvroKey
import org.apache.hadoop.io.NullWritable
...
object SparkRunner {
def readInput[S <: SpecificRecord: ClassTag](spark: SparkContext, source: Source[S]): RDD[S] = {
val job = Job.getInstance(spark.hadoopConfiguration)
Layout.each do |layout|
layout.update_version_types
end
class Layout
def update_version_types
versions.each do |version|
version.update_attribute('type', self.type) if version.type.nil?
end
end