-
-
Save CsBigDataHub/5b288af7f978cd57910462c3cae4c173 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import org.apache.spark.sql.SparkSession | |
import org.apache.spark.{SparkConf, SparkContext} | |
/** | |
* Created by vgiridatabricks on 8/19/16. | |
*/ | |
object WholeStageCodeGenExample { | |
def main(args: Array[String]): Unit = { | |
val spark = SparkSession | |
.builder() | |
.appName("Spark SQL Example") | |
.master("local") | |
.config("spark.sql.codegen.wholeStage", "true") //default is true so you don't have to setup this. | |
.getOrCreate() | |
// val df = spark.range(1L, 200000L, 1L, 1).toDF() | |
val startTime = System.nanoTime | |
val df = spark.range(1000L * 1000 * 1000).selectExpr("sum(id)").show() | |
val endTime = System.nanoTime | |
println(s"Total Execution Time " + (endTime - startTime).toDouble / 1000000000 + " seconds") | |
// df.show() | |
// df.explain() | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment