SparkのDataFrame/Datasetって型安全なの？ ref: https://qiita.com/piyo7/items/15a37e300e78795b09db

build.properties

sbt.version=1.2.6

build.sbt

javacOptions ++= Seq("-source", "1.8", "-target", "1.8")
scalaVersion := "2.12.7"
libraryDependencies ++= Seq(
  "org.apache.spark" %% "spark-core" % "2.4.0",
  "org.apache.spark" %% "spark-sql" % "2.4.0")

file0.scala

case class Person(name: String, height: Int)
val people: Dataset[Person]

file1.scala

import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
val spark = SparkSession.builder().master("local[*]").getOrCreate()
import spark.implicits._

case class Person(name: String, height: Int)
val people: Dataset[Person] = Seq(Person("Sayori", 157), Person("Natsuki", 149), Person("Yuri", 165)).toDS()

file10.txt

+-------+------+
|   name|height|
+-------+------+
| Sayori|   157|
|Natsuki|   149|
|   Yuri|   165|
+-------+------+

file11.scala

heights.select("name").show()

file12.txt

+-------+
|   name|
+-------+
| Sayori|
|Natsuki|
|   Yuri|
+-------+

file13.scala

case class Monika(m: Just[Monika], o: Just[Monika], n: Just[Monika], i: Just[Monika], k: Just[Monika], a: Just[Monika])
people.select("m", "o", "n", "i", "k", "a").as[Monika]

file14.scala

implicit class DataSetHelper[T](val dataset: Dataset[T]) extends AnyVal {
  def selectAs[U](implicit encoder: Encoder[U]): Dataset[U] =
      dataset.select(encoder.schema.fields.map(f => new ColumnName(f.name).cast(f.dataType)): _*).as[U]
}

file15.scala

people.selectAs[Monika]

file16.scala

val justMonika: Dataset[Person] = people.map(_.copy(name = "Monika"))

file17.scala

val justMonika: Dataset[Person] = people.filter(_.name == "Monika")

file2.txt

+-------+------+
|   name|height|
+-------+------+
| Sayori|   157|
|Natsuki|   149|
|   Yuri|   165|
+-------+------+

file3.scala

val names: DataFrame = people.select("name")

file4.txt

+-------+
|   name|
+-------+
| Sayori|
|Natsuki|
|   Yuri|
+-------+

file5.txt

[error] (run-main-2) org.apache.spark.sql.AnalysisException: cannot resolve '`age`' given input columns: [name, height];;
[error] 'Project ['age]
[error] +- LocalRelation [name#2, height#3]
[error] org.apache.spark.sql.AnalysisException: cannot resolve '`age`' given input columns: [name, height];;
[error] 'Project ['age]
[error] +- LocalRelation [name#2, height#3]

file6.scala

case class Height(height: Int)
val heights: Dataset[Height] = people.as[Height]

file7.scala

case class Age(age: Int)
val ages: Dataset[Age] = people.as[Age]

file8.txt

[error] (run-main-0) org.apache.spark.sql.AnalysisException: cannot resolve '`age`' given input columns: [name, height];
[error] org.apache.spark.sql.AnalysisException: cannot resolve '`age`' given input columns: [name, height];

file9.scala

case class Height(height: Int)
val heights: Dataset[Height] = people.as[Height]