Flatten Spark data frame fields structure, via SQL in Java. This fork also supports ArrayType fields.

flatten.java

class Toto
{
    public void Main()
    {
        final DataFrame source = GetDataFrame();
        final String querySelectSQL = flattenSchema(source.schema(), null);

        source.registerTempTable("source"); 
      
        final DataFrame flattenData = sqlContext.sql("SELECT " + querySelectSQL + " FROM source")
    }
    
    /**
    * Generate SQL to select columns as flat.
    */
    public String flattenSchema(StructType schema, String prefix)
    {
        final StringBuilder selectSQLQuery = new StringBuilder();

        for (StructField field : schema.fields())
        {
            final String fieldName = field.name();

            if (fieldName.startsWith("@"))
            {
                continue;
            }

            String colName = prefix == null ? fieldName : (prefix + "[0]." + fieldName);
            String colNameTarget = colName.replace("[0].", "_");
           
            DataType dtype = field.dataType();
            if (dtype.getClass().equals(ArrayType.class)) {
                   dtype = ((ArrayType) dtype).elementType();

            }
            if (field.dataType().getClass().equals(StructType.class))
            {
               selectSQLQuery.append(flattenSchema((StructType) dtype, colName));
            }
            else
            {
                selectSQLQuery.append(colName);
                selectSQLQuery.append(" as ");
                selectSQLQuery.append(colNameTarget);
            }

            selectSQLQuery.append(",");
        }

        if (selectSQLQuery.length() > 0)
        {
            selectSQLQuery.deleteCharAt(selectSQLQuery.length() - 1);
        }

        return selectSQLQuery.toString();
    }
}