spark_ui_example.py

sales = spark.read.option("header", True).csv("sales_train_evaluation.csv")
cols = sales.columns[6:]
groupby_state = sales \
  .selectExpr("id", "item_id", "dept_id", "cat_id", "store_id", "state_id",
              "stack({}, {}) as (d, amount)".format(len(cols), ', '.join(("'{}', {}".format(i, i) for i in cols)))) \
  .groupBy("state_id") \
  .agg(F.sum("amount").alias('amt_tot')) \
  .orderBy(F.col("amt_tot").desc())
groupby_state.show()