[iterate over rdd rows] how-to iterate over RDD rows and get DataFrame in scala spark #scala #spark

iterate-over-rdd-rows.scala

    // note if you could implement withColumn + udf it has been usually over 10x faster ... 
 
    val rddRows: RDD[Row] =
      inDf.rdd.map(row => {
      
        val lstRow = row.toSeq.toList
        var lstRowNew = lstRow
        // do stuff on the new lstRow here 

        Row.fromSeq(lstRowNew)
      })

    val dfOut = spark.createDataFrame(rddRows, inDf.schema)