spark_sql_example.scala

import org.apache.spark.sql._

// Create a Spark Session
val spark = SparkSession.builder().appName("test").master("local").getOrCreate()

// read some text source file
val srcDF = spark.read.format("csv").option("header", "true").option("inferSchema", "true").load("/home/jovyan/sales_src.csv")

// self explainatory i guess ? multiply Units Sold column by 2
val unitsBy2 = srcDF.withColumn("Units Sold", $"Units Sold" * 2) // transformation

// Filter rows by order id
val filterOrderId = unitsBy2.filter($"Order Id" > 100) // transformation

// select only 
val select = filterOrderId.select($"Region") // transformation

select.take(10) // action

select.explain(extended=true) // spark, please tell me what you did under the hood