Skip to content

Instantly share code, notes, and snippets.

@melrief
Last active November 27, 2019 19:10
Show Gist options
  • Save melrief/84ee65cd4f2c06e236df to your computer and use it in GitHub Desktop.
Save melrief/84ee65cd4f2c06e236df to your computer and use it in GitHub Desktop.
Problem of Spark with FunSuite and defaultParallelism
name := "Fun Suite Test"
scalaVersion := "2.10.4"
libraryDependencies ++= Seq(
"org.apache.spark" % "spark-core_2.10" % "1.0.2"
, "org.scalatest" % "scalatest_2.10" % "2.2.2"
)
import org.scalatest.FunSuite
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
class MyTestSuite extends FunSuite {
val conf = new SparkConf()
.setAppName("My Spark test")
.setMaster("local")
.set("spark.default.parallelism","1")
val sc = new SparkContext(conf)
try {
this.test("My Spark test") {
sc.parallelize(Seq(1,2,3)) // crashes
//sc.parallelize(Seq(1,2,3),4) // doesn't crash
}
} finally {
sc.stop
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment