Skip to content

Instantly share code, notes, and snippets.

@nitay

nitay/gist:d08cb294ccf00b80c49a Secret

Last active Sep 10, 2015
Embed
What would you like to do?
case class Foo(id: Int)
class Apache15SetTest extends Specification {
val sparkContext = new SparkContext(s"local[3]", "apache-1.5-test", new SparkConf)
val sqlContext = new SQLContext(sparkContext)
"ApacheSetTest" should {
"intersect rows" in {
val simpleRows1 = (1 to 100).map { id => Row(id) }
val simpleRows2 = (1 to 100).map { id => Row(id) }
val simpleSchema = StructType(StructField("i1", IntegerType) :: Nil)
val rdd1 = sparkContext.parallelize(simpleRows1)
val rdd2 = sparkContext.parallelize(simpleRows2)
val dataFrame1 = sqlContext.createDataFrame(rdd1, simpleSchema)
val dataFrame2 = sqlContext.createDataFrame(rdd2, simpleSchema)
val intersectDF = dataFrame1.intersect(dataFrame2)
intersectDF.count must beEqualTo(100)
}
"intersect case class" in {
import sqlContext.implicits._
val simpleRows1 = (1 to 100).map { id => Foo(id) }
val simpleRows2 = (1 to 100).map { id => Foo(id) }
val dataFrame1 = simpleRows1.toDF()
val dataFrame2 = simpleRows2.toDF()
val intersectDF = dataFrame1.intersect(dataFrame2)
intersectDF.count must beEqualTo(100)
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.