satendrakumar

//Producer 
import java.util.Properties
import java.util.concurrent.Future

import org.apache.kafka.clients.producer.{ProducerRecord, RecordMetadata}
import org.apache.kafka.common.serialization.StringSerializer

satendrakumar

import org.apache.spark.sql.{DataFrame, SQLContext}
import org.apache.spark.{SparkConf, SparkContext}

case class Demo(id: Int, name: String, info: Map[String, String], inner: Inner)

case class Inner(firstName: String)

object ParquetReadWrite extends App {

satendrakumar

import javax.inject.Inject

import com.google.inject._

import scala.collection.JavaConversions._


object DemoApp extends App {

satendrakumar

  

def withRetry[T](retryCount: Int)(t: => T): T =
    try
      t
    catch {
      case NonFatal(th) =>
        th.printStackTrace()
        if (retryCount >= 0) {
          println("Retrying............... [remaining count: [${retryCount-1}`]")

satendrakumar

import play.api.libs.json._


object JsonFormatter {
  
 implicit val residentReads = Json.reads[Resident]
  
}

case class Resident(name: String, age: Int, role: Option[String])

satendrakumar

akka {
  actor {
    default-dispatcher {
    
      fork-join-executor {

        parallelism-factor = 8

        parallelism-min = 16

satendrakumar

import org.apache.spark.sql.functions._
import org.apache.spark.sql.SparkSession

object DataFrameWithFileNameApp extends App {
  
  val spark: SparkSession =
    SparkSession
      .builder()
      .appName("DataFrameApp")
      .config("spark.master", "local[*]")

satendrakumar

<configuration>
    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <pattern>%highlight([%level]) - [%thread] - [%date] - [%logger] %msg%n</pattern>
        </encoder>
    </appender>
    <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <prudent>true</prudent>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>logs/etl-%d{yyyy-MM-dd}.log</fileNamePattern>

satendrakumar

import com.univocity.parsers.csv.{CsvParser, CsvParserSettings}


class CSVParser(delimiter: Char = ',') {

  private val parser = {
    val settings = new CsvParserSettings
    val format = settings.getFormat
    format.setLineSeparator("\n")
    format.setDelimiter(delimiter)

satendrakumar

import org.slf4j.{Logger, LoggerFactory}

trait Logging {

  protected val logger: Logger = LoggerFactory.getLogger(this.getClass())

  protected def debug(message: String): Unit = logger.debug(message)

  protected def debug(message: String, exception: Throwable): Unit = logger.debug(message, exception)