Skip to content

Instantly share code, notes, and snippets.

@dos65
Created January 14, 2019 11:44
Show Gist options
  • Save dos65/019eeb55ff9b41e4d192973cd2db67d3 to your computer and use it in GitHub Desktop.
Save dos65/019eeb55ff9b41e4d192973cd2db67d3 to your computer and use it in GitHub Desktop.
{ "name" : "x", "age": 10 }
{ "name" : "y", "age": 20 }
{ "name" : "z", "age": 30 }
from mistpy.decorators import *
@with_args(arg('path', type_hint=str))
@on_spark_session
def session_example(spark, path):
df = spark.read.json(path)
df.printSchema()
df.registerTempTable("people")
df.show()
df2 = spark.sql("SELECT age, name AS avg_age FROM people where age > 10")
df2.show()
rows = df2.collect()
result = [r.asDict(True) for r in rows]
return {"result": result}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment