ConfigUtils.scala

object ConfigUtils {
  
  def loadEnvProperties(environment : String) : Config ={
    import com.typesafe.config._
    val appConf = ConfigFactory.load()
    appConf.getConfig(environment)
  }
  
  def getSparkSession(devProps :Config) : SparkSession = {
    
     /*
      * Cassandra options
      */  
     val c_hosts = devProps.getString("cassandra.hosts")
     val c_portNumber = devProps.getInt("cassandra.portNumber") 
     val c_userName = devProps.getString("cassandra.username")
     val c_passwd = devProps.getString("cassandra.password")
     val c_concurrent_writes = devProps.getInt("cassandra.concurrent.writes")
     
      val c_options_conf : Map[String,String] = Map(
       "spark.cassandra.connection.host" -> c_hosts ,
       "spark.cassandra.connection.port" -> c_portNumber.toString(),
       "spark.cassandra.auth.username"-> c_userName,
       "spark.cassandra.auth.password" -> c_passwd,
       "spark.cassandra.output.concurrent.writes" -> c_concurrent_writes.toString()
       )
         
     import org.apache.spark.sql.cassandra._
    
    //2. spark factory - creatSparkSession
     
       val conf = new SparkConf(true)    
                      .set("spark.cassandra.connection.host",c_hosts)
                      .set("spark.cassandra.connection.port",c_portNumber.toString())
                      .set("spark.cassandra.auth.username",c_userName)
                      .set("spark.cassandra.auth.password",c_passwd)
                      .set("spark.cassandra.output.concurrent.writes" ,c_concurrent_writes.toString())
    
     val spark = SparkSession
        .builder()
        .appName("DatabaseMigrationUtility")
        .config("spark.master",devProps.getString("deploymentMaster"))
        .config("spark.dynamicAllocation.enabled",devProps.getString("spark.dynamicAllocation.enabled"))
        .config("spark.executor.memory",devProps.getString("spark.executor.memory"))
        .config("spark.executor.cores",devProps.getString("spark.executor.cores"))
        .config("spark.executor.instances",devProps.getString("spark.executor.instances"))
        .config(conf)   //I need to set this inorder to support RDD saving to Cassandra table
        .getOrCreate()
        .setCassandraConf(c_options_conf)

       spark
  }
  
  
}

  .config(conf)   //I need to set this inorder to support RDD saving to Cassandra table
    .getOrCreate()
    .setCassandraConf(c_options_conf) // as per spark sql 2.3.x i have set this.