Codegen dies (Spark 2.0.2 and 2.1.1) - no udf nesting
| import spark.implicits._ | |
| import org.apache.spark.sql.functions.udf | |
| import org.apache.spark.sql._ | |
| import org.apache.spark.sql.functions._ | |
| import org.apache.spark.sql.execution.debug._ | |
| val u = udf((a: Int) => a) | |
| val df = spark.sparkContext.parallelize(Seq(0)).toDF("0") | |
| val res = (1 until 20).foldLeft(df) { case (d, i) => | |
| val inputs = d.columns.toSeq.takeRight(1).map(col(_)) | |
| d.select(col("*"), u(inputs: _*).as(i.toString)) | |
| } | |
| res.debugCodegen() | |
| res.debug() | |
| res.show(false) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment