Load and query data from S3 in Zeppelin (Spark)

load_and_query.scala

// Load two data paths
val df1 = spark.read.load("/path/to/data1") // e.g. parquet files
val df2 = spark.read.load("/path/to/data2") // e.g. parquet files


// Union into single dataframe 
df1.createOrReplaceTempView("data1")
df2.createOrReplaceTempView("data2")

val df = spark.sql("""
SELECT * FROM data1
UNION ALL
SELECT * FROM data2
""")

df.registerOrReplaceTempView("data")

// Run a query
val query = spark.sql("SELECT count(*) FROM data")
query.show()