Skip to content

Instantly share code, notes, and snippets.

@DmytroMitin
Created April 19, 2023 11:17
Show Gist options
  • Save DmytroMitin/48f0fe84ddb44a8f53ee46f7b3dc8744 to your computer and use it in GitHub Desktop.
Save DmytroMitin/48f0fe84ddb44a8f53ee46f7b3dc8744 to your computer and use it in GitHub Desktop.
import com.fasterxml.jackson.core.Version
import com.fasterxml.jackson.databind.{Module, ObjectMapper}
import org.apache.spark.sql.SparkSession
object App {
val spark = SparkSession.builder
.master("local")
.appName("Spark app")
.getOrCreate()
import spark.implicits._
val kafkaStream = spark
.readStream
.format("kafka")
// .option("kafka.bootstrap.servers", settings.kafka.brokers)
.option("startingOffsets", "latest")
.option("failOnDataLoss", "false")
.option("subscribe", "serviceproblems")
.load()
val dataset = kafkaStream.select($"key", $"value").as[(String, String)]
val mapper = new ObjectMapper
mapper.registerModule(new ServiceProblemDeserializerModule())
case class ServiceProblemDeserializerModule() extends Module {
override def getModuleName: String = ???
override def version(): Version = ???
override def setupModule(context: Module.SetupContext): Unit = ???
}
}
ThisBuild / version := "0.1.0-SNAPSHOT"
//ThisBuild / scalaVersion := "2.12.0"
ThisBuild / scalaVersion := "2.11.8"
lazy val root = (project in file("."))
.settings(
name := "scalademo77"
)
//libraryDependencies += "org.apache.spark" %% "spark-sql" % "3.2.2"
libraryDependencies += "org.apache.spark" %% "spark-sql" % "2.2.3"
/media/data/jdk1.8.0_351/bin/java -server -Xmx1536M -Dsbt.supershell=false -Didea.managed=true -Dfile.encoding=UTF-8 -Didea.installation.dir=/media/data/idea-IU-211.7628.21 -jar /home/dmitin/.local/share/JetBrains/IntelliJIdea2023.1/Scala/launcher/sbt-launch.jar early(addPluginSbtFile=\"\"\"/tmp/idea.sbt\"\"\") "; set ideaPort in Global := 36853 ; idea-shell"
[info] welcome to sbt 1.8.2 (Oracle Corporation Java 1.8.0_351)
[info] loading global plugins from /home/dmitin/.sbt/1.0/plugins
[info] loading settings for project scalademo77-build from idea.sbt ...
[info] loading project definition from /media/data/Projects1/scalademo77/project
[warn] Unrecognized repository Scala Plugin Bundled Repository, ignoring it
[warn] Unrecognized repository Scala Plugin Bundled Repository, ignoring it
[info] loading settings for project root from build.sbt ...
[info] set current project to scalademo77 (in build file:/media/data/Projects1/scalademo77/)
[info] Defining Global / ideaPort
[info] The new value will be used by Compile / compile, Test / compile
[info] Reapplying settings...
[info] set current project to scalademo77 (in build file:/media/data/Projects1/scalademo77/)
[IJ]clean
[success] Total time: 0 s, completed 19.04.2023 14:16:57
[IJ]compile
[info] compiling 1 Scala source to /media/data/Projects1/scalademo77/target/scala-2.11/classes ...
[info] Non-compiled module 'compiler-bridge_2.11' for Scala 2.11.8. Compiling...
[info] Compilation completed in 8.309s.
[info] done compiling
[success] Total time: 14 s, completed 19.04.2023 14:17:18
[IJ]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment