Skip to content

Instantly share code, notes, and snippets.

@dondrake
Last active September 2, 2016 00:15
Show Gist options
  • Save dondrake/c136d61503b819f0643f8c02854a9cdf to your computer and use it in GitHub Desktop.
Save dondrake/c136d61503b819f0643f8c02854a9cdf to your computer and use it in GitHub Desktop.
case class C1(f1: String, f2: String, f3: String, f4: String)
case class C2(g1: String, g2: String, g3: String, g4: String)
case class C3(h1: String, h2: String, h3: String, h4: String)
val sqlContext = spark.sqlContext
val c1 = sc.parallelize(Seq(
C1("h1", "c1a1", "c1b1", "c1c1"),
C1("h2", "c1a2", "c1b2", "c1c2"),
C1(null, "c1a3", "c1b3", "c1c3")
)).toDF
c1.createOrReplaceTempView("c1")
val c2 = sc.parallelize(Seq(
C2("h1", "c2a1", "c2b1", "c2c1"),
C2("h2", "c2a2", "c2b2", "c2c2"),
C2(null, "c2a3", "c2b3", "c2c3"),
C2(null, "c2a4", "c2b4", "c2c4"),
C2("h333", "c2a333", "c2b333", "c2c333")
)).toDF
c2.createOrReplaceTempView("c2")
val c3 = sc.parallelize(Seq(
C3("h1", "c3a1", "c3b1", "c3c1"),
C3("h2", "c3a2", "c3b2", "c3c2"),
C3(null, "c3a3", "c3b3", "c3c3")
)).toDF
c3.createOrReplaceTempView("c3")
// doesn't work in Spark 2.0, works in Spark 1.6
val bad_df = sqlContext.sql("""
select *
from c1, c3
left outer join c2 on (c1.f1 = c2.g1)
where c1.f1 = c3.h1
""").show()
// works in both
val works_df = sqlContext.sql("""
select *
from c1
left outer join c2 on (c1.f1 = c2.g1),
c3
where c1.f1 = c3.h1
""").show()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment