App.scala

import com.fasterxml.jackson.core.Version
import com.fasterxml.jackson.databind.{Module, ObjectMapper}
import org.apache.spark.sql.SparkSession

object App {
  val spark = SparkSession.builder
    .master("local")
    .appName("Spark app")
    .getOrCreate()

  import spark.implicits._
  val kafkaStream = spark
    .readStream
    .format("kafka")
//    .option("kafka.bootstrap.servers", settings.kafka.brokers)
    .option("startingOffsets", "latest")
    .option("failOnDataLoss", "false")
    .option("subscribe", "serviceproblems")
    .load()

  val dataset = kafkaStream.select($"key", $"value").as[(String, String)]
  val mapper = new ObjectMapper
  mapper.registerModule(new ServiceProblemDeserializerModule())

  case class ServiceProblemDeserializerModule() extends Module {
    override def getModuleName: String = ???
    override def version(): Version = ???
    override def setupModule(context: Module.SetupContext): Unit = ???
  }
}

build.sbt

ThisBuild / version := "0.1.0-SNAPSHOT"

ThisBuild / scalaVersion := "2.12.1"
//ThisBuild / scalaVersion := "2.12.0"
//ThisBuild / scalaVersion := "2.11.8"

lazy val root = (project in file("."))
  .settings(
    name := "scalademo77"
  )

libraryDependencies += "org.apache.spark" %% "spark-sql" % "3.2.2"
//libraryDependencies += "org.apache.spark" %% "spark-sql" % "2.2.3"

log.txt

/usr/local/jdk1.8.0_162/bin/java -server -Xmx1536M -Dsbt.supershell=false -Didea.managed=true -Dfile.encoding=UTF-8 -Didea.installation.dir=/media/data/idea-IU-211.7628.21 -jar /home/dmitin/.local/share/JetBrains/IntelliJIdea2023.1/Scala/launcher/sbt-launch.jar early(addPluginSbtFile=\"\"\"/tmp/idea175.sbt\"\"\") "; set ideaPort in Global := 36853 ; idea-shell"
[info] welcome to sbt 1.8.2 (Oracle Corporation Java 1.8.0_162)
[info] loading global plugins from /home/dmitin/.sbt/1.0/plugins
[info] loading project definition from /media/data/Projects1/scalademo77/project/project
[info] loading settings for project scalademo77-build from plugins.sbt,idea175.sbt ...
[info] loading project definition from /media/data/Projects1/scalademo77/project
[warn] Unrecognized repository Scala Plugin Bundled Repository, ignoring it
[warn] Unrecognized repository Scala Plugin Bundled Repository, ignoring it
[info] loading settings for project root from build.sbt ...
[info] set current project to scalademo77 (in build file:/media/data/Projects1/scalademo77/)
[info] Defining Global / ideaPort
[info] The new value will be used by Compile / compile, Test / compile
[info] Reapplying settings...
[info] set current project to scalademo77 (in build file:/media/data/Projects1/scalademo77/)
[IJ]clean
[success] Total time: 0 s, completed 19.04.2023 16:42:24
[IJ]compile
[info] compiling 1 Scala source to /media/data/Projects1/scalademo77/target/scala-2.12/classes ...
[info] done compiling
[success] Total time: 6 s, completed 19.04.2023 16:42:34
[IJ]