Skip to content

Instantly share code, notes, and snippets.

View shatestest's full-sized avatar

Shyam shatestest

  • Steelwedge
  • India
View GitHub Profile
@shatestest
shatestest / DataFrameReaderOptions
Last active May 3, 2019 13:32
Issue with setting jdbc/partitionColumn type to Date in spark 2.4.1
Map(driver -> oracle.jdbc.OracleDriver,
numPartitions -> 2000,
timestampFormat -> yyyy-mm-dd hh:mm:ss,
upperBound -> 2019-05-01 23:59:59.0,
lowerBound -> 2002-03-31 00:00:00.0,
partitionColumn -> DATA_DATE,
url -> jdbc:oracle:thin:@,
user -> user1, password -> pwd1,
customSchema -> DATA_DATE TIMESTAMP, DATA_ITEM_VALUE_NUMERIC DECIMAL(38, 15) , MODEL_FAMILY_ID INTEGER , FISCAL_YEAR INTEGER , FISCAL_QUARTER INTEGER,
fetchsize -> 10000)
object {
@throws(classOf[Exception])
def process(ymlFilename:String):Option[QueryEntities] = {
val ymlFilePath = URLDecoder.decode(ymlFilename, "UTF-8")
Try(Source.fromFile(ymlFilePath).mkString.parseYaml) match {
case Success(yml) => {
object ConfigUtils {
def loadEnvProperties(environment : String) : Config ={
import com.typesafe.config._
val appConf = ConfigFactory.load()
appConf.getConfig(environment)
}
def getSparkSession(devProps :Config) : SparkSession = {
package oracle2cassandra.scala
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.DataFrameReader
import com.snp.helpers.Model._
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import scala.collection.Seq
import java.io.IOException
import com.snp.utils.DbUtils
trait Processor {
def process()=println("default process")
}
case class ProcessorOne() extends Processor
case class ProcessorTwo() extends Processor {
override def process=println("process 2")
}
case class ProcessorThree() extends Processor {
override def process=println("process 3")
}