Skip to content

Instantly share code, notes, and snippets.

@AndriiStefaniv
Created October 26, 2018 11:42
Show Gist options
  • Save AndriiStefaniv/902938c71440c3f25769528e5c730d4f to your computer and use it in GitHub Desktop.
Save AndriiStefaniv/902938c71440c3f25769528e5c730d4f to your computer and use it in GitHub Desktop.
trait Processor {
def process(oraOptionDfConfig: DataFrameReader, sparkSession: SparkSession): Unit
}
class BMValsProcessor extends Processor {
def process(oraOptionDfConfig: DataFrameReader, sparkSession: SparkSession): Unit = {
println("in BMValsProcessor") ///actual business logic
}
}
class BMValsProcessor2 extends Processor {
def process(oraOptionDfConfig: DataFrameReader, sparkSession: SparkSession): Unit = {
println("in BMValsProcessor2") ///actual business logic
}
}
object ValProcessor {
def main(args: Array[String]): Unit = {
val df: DataFrameReader = null; // initialization of DataFrameReader
val spark: SparkSession = null; // initialization of SparkSession
val procs: Map[String, (DataFrameReader, SparkSession) => Unit] = getAllDefinedProcessors()
procs.values.foreach(
proc => proc(df, spark)
)
}
def getAllDefinedProcessors(): Map[String, (DataFrameReader, SparkSession) => Unit] = {
Map(
"bm_vals" -> new BMValsProcessor().process,
"bm_vals2" -> new BMValsProcessor2().process
)
}
}
@shatestest
Copy link

shatestest commented Nov 1, 2018

trait Processor {
def processOne(oraOptionDfConfig: DataFrameReader, sparkSession: SparkSession): Unit
def processTwo(oraOptionDfConfig: DataFrameReader, sparkSession: SparkSession): Unit
}

class BMValsProcessor extends Processor {
def processOne (oraOptionDfConfig: DataFrameReader, sparkSession: SparkSession): Unit = {
println("in BMValsProcessor processOne ") ///actual business logic
}
def processTwo (oraOptionDfConfig: DataFrameReader, sparkSession: SparkSession): Unit = {
println("in BMValsProcessor processTwo ") ///actual business logic
}
}

class BMValsProcessor2 extends Processor {
def processOne(oraOptionDfConfig: DataFrameReader, sparkSession: SparkSession): Unit = {
println("in BMValsProcessor2 processOne") ///actual business logic
}
def processTwo(oraOptionDfConfig: DataFrameReader, sparkSession: SparkSession): Unit = {
println("in BMValsProcessor2 processTwo") ///actual business logic
}
}

object ValProcessor {
def main(args: Array[String]): Unit = {

val df: DataFrameReader = null; // initialization of DataFrameReader
val spark: SparkSession = null; // initialization of SparkSession

val procs: Map[String, (DataFrameReader, SparkSession) => Unit] = getAllDefinedProcessors()

procs.values.foreach(
  proc => proc(df, spark)
)

}

def getAllDefinedProcessors(): Map[String, (DataFrameReader, SparkSession) => Unit] = {

Map(
  "bm_vals" -> new BMValsProcessor().processOne,  /// How to register/point and call processTwo
  "bm_vals2" -> new BMValsProcessor2().processOne /// How to register/point and call processTwo
)

}

}

@AndriiStefaniv
Copy link
Author

AndriiStefaniv commented Nov 1, 2018

You could try something like

trait Processor {
  def processOne(oraOptionDfConfig: DataFrameReader, sparkSession: SparkSession): Unit

  def processTwo(oraOptionDfConfig: DataFrameReader, sparkSession: SparkSession): Unit
}

class BMValsProcessor extends Processor {
  def processOne(oraOptionDfConfig: DataFrameReader, sparkSession: SparkSession): Unit = {
    println("in BMValsProcessor processOne ") ///actual business logic
  }

  def processTwo(oraOptionDfConfig: DataFrameReader, sparkSession: SparkSession): Unit = {
    println("in BMValsProcessor processTwo ") ///actual business logic
  }
}

class BMValsProcessor2 extends Processor {
  def processOne(oraOptionDfConfig: DataFrameReader, sparkSession: SparkSession): Unit = {
    println("in BMValsProcessor2 processOne") ///actual business logic
  }

  def processTwo(oraOptionDfConfig: DataFrameReader, sparkSession: SparkSession): Unit = {
    println("in BMValsProcessor2 processTwo") ///actual business logic
  }
}

object ValProcessor {
  def main(args: Array[String]): Unit = {

    val df: DataFrameReader = null; // initialization of DataFrameReader
    val spark: SparkSession = null; // initialization of SparkSession

    val procs: Map[String, Processor] = getAllDefinedProcessors()

    procs.values.foreach {
      proc =>
        proc.processOne(df, spark)
        proc.processTwo(df, spark)
    }
  }

  def getAllDefinedProcessors(): Map[String, Processor] = {

    Map(
      "bm_vals" -> new BMValsProcessor(), 
      "bm_vals2" -> new BMValsProcessor2() 
    )
  }

}

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment