Spark UDF change

udf_diff_spark

import org.apache.spark.sql.types._
import org.apache.spark.sql.functions._
val f = udf((x: Int) => x, IntegerType)

val df = Seq((None), (Some(1))).toDF("value")

df.show
/* In both Spark versions 2.X and 3.X
+-----+
|value|
+-----+
| null|
|    1|
+-----+
*/

df.select(f($"value").as("incremented")).show

/* Spark3 with option spark.sql.legacy.allowUntypedScalaUDF=true
+-----------+
|incremented|
+-----------+
|          0|
|          1|
+-----------+
*/

/* Spark 2
-----------+
|incremented|
+-----------+
|       null|
|          1|
+-----------+
*/