Scala2 implicit conversion to achieve fluent interface on spark dataframes

build.sbt

ThisBuild / version := "0.1.0-SNAPSHOT"

ThisBuild / scalaVersion := "2.12.17"

libraryDependencies ++= Seq(
  "org.apache.spark" %% "spark-core" % "3.2.2",
  "org.apache.spark" %% "spark-sql" % "3.2.2",
)

Main.scala

import org.apache.spark.sql.functions.{col, lower}
import org.apache.spark.sql.{DataFrame, SparkSession}

object Main {

  def main(args: Array[String]): Unit = {

    // initialize
    val spark = SparkSession.builder.master("local[1]").appName("Simple Application").getOrCreate()

    // allows .toDf conversion
    import spark.implicits._

    // create sample DataFrame
    val columns = Seq("language", "users_count")
    val data = Seq(("Java", "20000"), ("Python", "100000"), ("Scala", "3000"))
    val sample_df: DataFrame = data.toDF(columns:_*)

    // print sample data frame
    println(
      sample_df
        .show
    )
    // print data frame after modifying it
    println(
      sample_df
        .langToLowercase
        .renameLangColumn
        .show
    )
  }

  implicit class Functions(df: DataFrame) {
    def langToLowercase: DataFrame = df.withColumn("language", lower(col("language")))
    def renameLangColumn: DataFrame = df.withColumnRenamed("language", "language_name")
  }
}

notes.md

Implicit conversions allow extending DataFrame class/trait without inheritance. Code is converted to function calls during compilation.

For example:

sample_df
  .langToLowercase
  .renameLangColumn

is converted to: renameLangColumn(langToLowercase(sample_df))

Running this code outputs:

+--------+-----------+ |language|users_count| +--------+-----------+ | Java| 20000| | Python| 100000| | Scala| 3000| +--------+-----------+

+-------------+-----------+ |language_name|users_count| +-------------+-----------+ | java| 20000| | python| 100000| | scala| 3000| +-------------+-----------+

In case of error about StorageUtils see https://stackoverflow.com/questions/72230174/java-17-solution-for-spark-java-lang-noclassdeffounderror-could-not-initializ