Created
June 30, 2022 21:06
-
-
Save javamonkey79/eb981434b5a2647ba2529a821413ecab to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| == Parsed Logical Plan == | |
| Join FullOuter, (_2#37.key1 = _1#36.key1) | |
| :- Project [named_struct(key1, key1#23, value, value#24) AS _1#36] | |
| : +- Project [_1#18 AS key1#23, _2#19 AS value#24] | |
| : +- LocalRelation [_1#18, _2#19] | |
| +- Project [named_struct(key1, key1#7, values, values#8) AS _2#37] | |
| +- Project [_1#2 AS key1#7, _2#3 AS values#8] | |
| +- LocalRelation [_1#2, _2#3] | |
| == Analyzed Logical Plan == | |
| _1: struct<key1:string,value:int>, _2: struct<key1:string,values:array<struct<field_1:string,field_2:string,field_3:string,field_4:string,field_5:timestamp,field_6:date,field_7:string,field_8:string,field_9:string,field_10:string,field_11:int,field_12:string,field_13:string,field_14:string,field_15:string,field_16:string,field_17:string,field_18:double,field_19:double,field_20:double,field_21:double,field_22:double,field_23:double,field_24:double,... 57 more fields>>> | |
| Join FullOuter, (_2#37.key1 = _1#36.key1) | |
| :- Project [named_struct(key1, key1#23, value, value#24) AS _1#36] | |
| : +- Project [_1#18 AS key1#23, _2#19 AS value#24] | |
| : +- LocalRelation [_1#18, _2#19] | |
| +- Project [named_struct(key1, key1#7, values, values#8) AS _2#37] | |
| +- Project [_1#2 AS key1#7, _2#3 AS values#8] | |
| +- LocalRelation [_1#2, _2#3] | |
| == Optimized Logical Plan == | |
| Join FullOuter, (_2#37.key1 = _1#36.key1) | |
| :- LocalRelation [_1#36] | |
| +- LocalRelation [_2#37] | |
| == Physical Plan == | |
| SortMergeJoin [_1#36.key1], [_2#37.key1], FullOuter | |
| :- *(1) Sort [_1#36.key1 ASC NULLS FIRST], false, 0 | |
| : +- Exchange hashpartitioning(_1#36.key1, 200), ENSURE_REQUIREMENTS, [id=#12] | |
| : +- LocalTableScan [_1#36] | |
| +- Sort [_2#37.key1 ASC NULLS FIRST], false, 0 | |
| +- Exchange hashpartitioning(_2#37.key1, 200), ENSURE_REQUIREMENTS, [id=#13] | |
| +- LocalTableScan [_2#37] | |
| == Parsed Logical Plan == | |
| 'SerializeFromObject [if (isnull(unwrapoption(ObjectType(class scala.Tuple2), knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1))) null else named_struct(_1, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class scala.Tuple2), knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1))._1, true, false), _2, knownnotnull(unwrapoption(ObjectType(class scala.Tuple2), knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1))._2) AS _1#54, mapobjects(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127), if (isnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127))) null else named_struct(field_1, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_1), true, false), field_2, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_2), true, false), field_3, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_3), true, false), field_4, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_4), true, false), field_5, staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, TimestampType, fromJavaTimestamp, unwrapoption(ObjectType(class java.sql.Timestamp), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_5), true, false), field_6, staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, DateType, fromJavaDate, unwrapoption(ObjectType(class java.sql.Date), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_6), true, false), field_7, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_7), true, false), field_8, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_8), true, false), field_9, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_9), true, false), field_10, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_10), true, false), field_11, unwrapoption(IntegerType, knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_11), field_12, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_12), true, false), ... 138 more fields), knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2, None) AS _2#55] | |
| +- 'MapElements ProcessSpec$$Lambda$2764/00000000FB32F620@ad1d95f9, class scala.Tuple2, [StructField(_1,StructType(StructField(_1,StringType,true), StructField(_2,IntegerType,false)),true), StructField(_2,StructType(StructField(_1,StringType,true), StructField(_2,ArrayType(StructType(StructField(field_1,StringType,true), StructField(field_2,StringType,true), StructField(field_3,StringType,true), StructField(field_4,StringType,true), StructField(field_5,TimestampType,true), StructField(field_6,DateType,true), StructField(field_7,StringType,true), StructField(field_8,StringType,true), StructField(field_9,StringType,true), StructField(field_10,StringType,true), StructField(field_11,IntegerType,true), StructField(field_12,StringType,true), StructField(field_13,StringType,true), StructField(field_14,StringType,true), StructField(field_15,StringType,true), StructField(field_16,StringType,true), StructField(field_17,StringType,true), StructField(field_18,DoubleType,true), StructField(field_19,DoubleType,true), StructField(field_20,DoubleType,true), StructField(field_21,DoubleType,true), StructField(field_22,DoubleType,true), StructField(field_23,DoubleType,true), StructField(field_24,DoubleType,true), StructField(field_25,DoubleType,true), StructField(field_26,DoubleType,true), StructField(field_27,DoubleType,true), StructField(field_28,DoubleType,true), StructField(field_29,DoubleType,true), StructField(field_30,StringType,true), StructField(field_31,StringType,true), StructField(field_32,StringType,true), StructField(field_33,StringType,true), StructField(field_34,StringType,true), StructField(field_35,StringType,true), StructField(field_36,StringType,true), StructField(field_37,StringType,true), StructField(field_38,StringType,true), StructField(field_39,StringType,true), StructField(field_40,StringType,true), StructField(field_41,StringType,true), StructField(field_42,StringType,true), StructField(field_43,StringType,true), StructField(field_44,StringType,true), StructField(field_45,StringType,true), StructField(field_46,StringType,true), StructField(field_47,IntegerType,true), StructField(field_48,IntegerType,true), StructField(field_49,DateType,true), StructField(field_50,DateType,true), StructField(field_51,DateType,true), StructField(field_52,DateType,true), StructField(field_53,StringType,true), StructField(field_54,StringType,true), StructField(field_55,IntegerType,true), StructField(field_56,DateType,true), StructField(field_57,StringType,true), StructField(field_58,StringType,true), StructField(field_59,StringType,true), StructField(field_60,StringType,true), StructField(field_61,StringType,true), StructField(field_62,StringType,true), StructField(field_63,StringType,true), StructField(field_64,BooleanType,true), StructField(field_65,MapType(StringType,StringType,true),true), StructField(field_66,IntegerType,true), StructField(field_67,IntegerType,true), StructField(field_68,StringType,true), StructField(field_154,StructType(StructField(field_70,LongType,true), StructField(field_71,StringType,true), StructField(field_72,StringType,true), StructField(field_73,LongType,true), StructField(field_74,StringType,true), StructField(field_75,StringType,true), StructField(field_76,StringType,true), StructField(field_77,StringType,true), StructField(field_78,StringType,true), StructField(field_82,StructType(StructField(field_80,StringType,true), StructField(field_81,StringType,true), StructField(field_82,IntegerType,true)),true), StructField(field_86,StructType(StructField(field_84,IntegerType,true), StructField(field_85,IntegerType,true), StructField(field_86,IntegerType,true)),true), StructField(field_87,DoubleType,true), StructField(field_88,DoubleType,true), StructField(field_94,ArrayType(StructType(StructField(field_90,StringType,true), StructField(field_91,StringType,true), StructField(field_92,StringType,true), StructField(field_93,DoubleType,true), StructField(field_94,DoubleType,true)),true),true), StructField(field_108,StructType(StructField(field_96,StringType,true), StructField(field_97,StringType,true), StructField(field_98,StringType,true), StructField(field_99,DoubleType,true), StructField(field_100,StringType,true), StructField(field_101,IntegerType,true), StructField(field_102,StringType,true), StructField(field_103,StringType,true), StructField(field_104,StringType,true), StructField(field_105,IntegerType,true), StructField(field_106,IntegerType,true), StructField(field_107,IntegerType,true), StructField(field_108,IntegerType,true)),true), StructField(field_109,StringType,true), StructField(field_119,StructType(StructField(field_111,StringType,true), StructField(field_112,StringType,true), StructField(field_113,StringType,true), StructField(field_114,StringType,true), StructField(field_115,StringType,true), StructField(field_116,StringType,true), StructField(field_117,StringType,true), StructField(field_118,StringType,true), StructField(field_119,StringType,true)),true), StructField(field_129,StructType(StructField(field_121,StringType,true), StructField(field_122,StringType,true), StructField(field_123,StringType,true), StructField(field_124,StringType,true), StructField(field_125,StringType,true), StructField(field_126,StringType,true), StructField(field_127,StringType,true), StructField(field_128,StringType,true), StructField(field_129,StringType,true)),true), StructField(field_139,StructType(StructField(field_131,StringType,true), StructField(field_132,StringType,true), StructField(field_133,StringType,true), StructField(field_134,StringType,true), StructField(field_135,StringType,true), StructField(field_136,StringType,true), StructField(field_137,StringType,true), StructField(field_138,StringType,true), StructField(field_139,StringType,true)),true), StructField(field_146,ArrayType(StructType(StructField(field_141,LongType,true), StructField(field_142,StringType,true), StructField(field_143,StringType,true), StructField(field_144,StringType,true), StructField(field_145,LongType,true), StructField(field_146,LongType,true)),true),true), StructField(field_147,ArrayType(StringType,true),true), StructField(field_148,ArrayType(StringType,true),true), StructField(field_154,ArrayType(StructType(StructField(field_150,LongType,true), StructField(field_151,StringType,true), StructField(field_152,StringType,true), StructField(field_153,StringType,true), StructField(field_154,ArrayType(StringType,true),true)),true),true)),true), StructField(field_167,StructType(StructField(field_165,StructType(StructField(field_157,DoubleType,true), StructField(field_158,DoubleType,true), StructField(field_159,DoubleType,true), StructField(field_160,DoubleType,true), StructField(field_161,DoubleType,true), StructField(field_162,DecimalType(38,18),true), StructField(field_163,DecimalType(38,18),true), StructField(field_164,DoubleType,true), StructField(field_165,DoubleType,true)),true), StructField(field_166,LongType,true), StructField(field_167,MapType(StringType,StringType,true),true)),true), StructField(field_267,StructType(StructField(field_172,StructType(StructField(field_170,DoubleType,true), StructField(field_171,DoubleType,true), StructField(field_172,StringType,true)),true), StructField(field_191,StructType(StructField(field_174,DoubleType,true), StructField(field_175,DoubleType,true), StructField(field_176,DoubleType,true), StructField(field_177,DoubleType,true), StructField(field_178,DoubleType,true), StructField(field_179,DoubleType,true), StructField(field_180,DoubleType,true), StructField(field_181,DoubleType,true), StructField(field_182,DoubleType,true), StructField(field_183,DoubleType,true), StructField(field_184,DoubleType,true), StructField(field_185,DoubleType,true), StructField(field_186,DoubleType,true), StructField(field_187,IntegerType,true), StructField(field_188,LongType,true), StructField(field_189,LongType,true), StructField(field_190,LongType,true), StructField(field_191,LongType,true)),true), StructField(field_198,StructType(StructField(field_193,DoubleType,true), StructField(field_194,DoubleType,true), StructField(field_195,DoubleType,true), StructField(field_196,DoubleType,true), StructField(field_197,DoubleType,true), StructField(field_198,DoubleType,true)),true), StructField(field_203,StructType(StructField(field_200,DecimalType(38,18),true), StructField(field_201,DoubleType,true), StructField(field_202,DecimalType(38,18),true), StructField(field_203,IntegerType,true)),true), StructField(field_228,StructType(StructField(field_205,DoubleType,true), StructField(field_206,DoubleType,true), StructField(field_207,DoubleType,true), StructField(field_208,DoubleType,true), StructField(field_209,StringType,true), StructField(field_228,StructType(StructField(field_211,IntegerType,true), StructField(field_212,StringType,true), StructField(field_213,DoubleType,true), StructField(field_214,IntegerType,true), StructField(field_215,DoubleType,true), StructField(field_216,IntegerType,true), StructField(field_217,DoubleType,true), StructField(field_218,IntegerType,true), StructField(field_219,IntegerType,true), StructField(field_220,IntegerType,true), StructField(field_221,IntegerType,true), StructField(field_222,StringType,true), StructField(field_223,DateType,true), StructField(field_224,IntegerType,true), StructField(field_225,IntegerType,true), StructField(field_226,IntegerType,true), StructField(field_227,IntegerType,true), StructField(field_228,StringType,true)),true)),true), StructField(field_247,ArrayType(StructType(StructField(field_230,TimestampType,true), StructField(field_231,LongType,true), StructField(field_232,StringType,true), StructField(field_233,StringType,true), StructField(field_234,StringType,true), StructField(field_235,TimestampType,true), StructField(field_236,TimestampType,true), StructField(field_237,DoubleType,true), StructField(field_238,IntegerType,true), StructField(field_239,IntegerType,true), StructField(field_240,BooleanType,true), StructField(field_241,IntegerType,true), StructField(field_242,IntegerType,true), StructField(field_243,DoubleType,true), StructField(field_244,LongType,true), StructField(field_245,StringType,true), StructField(field_246,TimestampType,true), StructField(field_247,StringType,true)),true),true), StructField(field_266,ArrayType(StructType(StructField(field_249,TimestampType,true), StructField(field_250,LongType,true), StructField(field_251,StringType,true), StructField(field_252,StringType,true), StructField(field_253,StringType,true), StructField(field_254,TimestampType,true), StructField(field_255,TimestampType,true), StructField(field_256,DoubleType,true), StructField(field_257,IntegerType,true), StructField(field_258,IntegerType,true), StructField(field_259,BooleanType,true), StructField(field_260,IntegerType,true), StructField(field_261,IntegerType,true), StructField(field_262,DoubleType,true), StructField(field_263,LongType,true), StructField(field_264,StringType,true), StructField(field_265,TimestampType,true), StructField(field_266,StringType,true)),true),true), StructField(field_267,DecimalType(38,18),true)),true), StructField(field_274,ArrayType(StructType(StructField(field_269,StringType,true), StructField(field_270,StringType,true), StructField(field_271,StringType,true), StructField(field_272,StringType,true), StructField(field_273,DoubleType,true), StructField(field_274,StringType,true)),true),true), StructField(field_275,IntegerType,true), StructField(field_290,StructType(StructField(field_277,IntegerType,true), StructField(field_278,BooleanType,true), StructField(field_279,IntegerType,true), StructField(field_280,BooleanType,true), StructField(field_281,BooleanType,true), StructField(field_282,BooleanType,true), StructField(field_283,BooleanType,true), StructField(field_284,BooleanType,true), StructField(field_285,BooleanType,true), StructField(field_286,StringType,true), StructField(field_287,StringType,true), StructField(field_288,StringType,true), StructField(field_289,BooleanType,true), StructField(field_290,BooleanType,true)),true), StructField(field_306,StructType(StructField(field_296,StructType(StructField(field_293,BooleanType,true), StructField(field_294,BooleanType,true), StructField(field_295,StringType,true), StructField(field_296,StringType,true)),true), StructField(field_301,StructType(StructField(field_298,BooleanType,true), StructField(field_299,BooleanType,true), StructField(field_300,StringType,true), StructField(field_301,StringType,true)),true), StructField(field_306,StructType(StructField(field_303,BooleanType,true), StructField(field_304,BooleanType,true), StructField(field_305,StringType,true), StructField(field_306,StringType,true)),true)),true), StructField(field_307,IntegerType,true), StructField(field_308,BooleanType,true), StructField(field_309,BooleanType,true), StructField(field_319,StructType(StructField(field_311,LongType,true), StructField(field_312,LongType,true), StructField(field_313,BooleanType,true), StructField(field_314,IntegerType,true), StructField(field_315,StringType,true), StructField(field_316,StringType,true), StructField(field_317,BooleanType,true), StructField(field_318,BooleanType,true), StructField(field_319,BooleanType,true)),true), StructField(field_320,TimestampType,true), StructField(field_321,DateType,true)),true),true)),true)], obj#53: scala.Tuple2 | |
| +- 'DeserializeToObject unresolveddeserializer(newInstance(class scala.Tuple2)), obj#52: scala.Tuple2 | |
| +- Join FullOuter, (_2#37.key1 = _1#36.key1) | |
| :- Project [named_struct(key1, key1#23, value, value#24) AS _1#36] | |
| : +- Project [_1#18 AS key1#23, _2#19 AS value#24] | |
| : +- LocalRelation [_1#18, _2#19] | |
| +- Project [named_struct(key1, key1#7, values, values#8) AS _2#37] | |
| +- Project [_1#2 AS key1#7, _2#3 AS values#8] | |
| +- LocalRelation [_1#2, _2#3] | |
| == Analyzed Logical Plan == | |
| _1: struct<_1:string,_2:int>, _2: array<struct<field_1:string,field_2:string,field_3:string,field_4:string,field_5:timestamp,field_6:date,field_7:string,field_8:string,field_9:string,field_10:string,field_11:int,field_12:string,field_13:string,field_14:string,field_15:string,field_16:string,field_17:string,field_18:double,field_19:double,field_20:double,field_21:double,field_22:double,field_23:double,field_24:double,... 57 more fields>> | |
| SerializeFromObject [if (isnull(unwrapoption(ObjectType(class scala.Tuple2), knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1))) null else named_struct(_1, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class scala.Tuple2), knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1))._1, true, false), _2, knownnotnull(unwrapoption(ObjectType(class scala.Tuple2), knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1))._2) AS _1#54, mapobjects(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127), if (isnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127))) null else named_struct(field_1, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_1), true, false), field_2, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_2), true, false), field_3, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_3), true, false), field_4, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_4), true, false), field_5, staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, TimestampType, fromJavaTimestamp, unwrapoption(ObjectType(class java.sql.Timestamp), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_5), true, false), field_6, staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, DateType, fromJavaDate, unwrapoption(ObjectType(class java.sql.Date), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_6), true, false), field_7, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_7), true, false), field_8, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_8), true, false), field_9, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_9), true, false), field_10, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_10), true, false), field_11, unwrapoption(IntegerType, knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_11), field_12, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, 127)).field_12), true, false), ... 138 more fields), knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2, None) AS _2#55] | |
| +- MapElements ProcessSpec$$Lambda$2764/00000000FB32F620@ad1d95f9, class scala.Tuple2, [StructField(_1,StructType(StructField(_1,StringType,true), StructField(_2,IntegerType,false)),true), StructField(_2,StructType(StructField(_1,StringType,true), StructField(_2,ArrayType(StructType(StructField(field_1,StringType,true), StructField(field_2,StringType,true), StructField(field_3,StringType,true), StructField(field_4,StringType,true), StructField(field_5,TimestampType,true), StructField(field_6,DateType,true), StructField(field_7,StringType,true), StructField(field_8,StringType,true), StructField(field_9,StringType,true), StructField(field_10,StringType,true), StructField(field_11,IntegerType,true), StructField(field_12,StringType,true), StructField(field_13,StringType,true), StructField(field_14,StringType,true), StructField(field_15,StringType,true), StructField(field_16,StringType,true), StructField(field_17,StringType,true), StructField(field_18,DoubleType,true), StructField(field_19,DoubleType,true), StructField(field_20,DoubleType,true), StructField(field_21,DoubleType,true), StructField(field_22,DoubleType,true), StructField(field_23,DoubleType,true), StructField(field_24,DoubleType,true), StructField(field_25,DoubleType,true), StructField(field_26,DoubleType,true), StructField(field_27,DoubleType,true), StructField(field_28,DoubleType,true), StructField(field_29,DoubleType,true), StructField(field_30,StringType,true), StructField(field_31,StringType,true), StructField(field_32,StringType,true), StructField(field_33,StringType,true), StructField(field_34,StringType,true), StructField(field_35,StringType,true), StructField(field_36,StringType,true), StructField(field_37,StringType,true), StructField(field_38,StringType,true), StructField(field_39,StringType,true), StructField(field_40,StringType,true), StructField(field_41,StringType,true), StructField(field_42,StringType,true), StructField(field_43,StringType,true), StructField(field_44,StringType,true), StructField(field_45,StringType,true), StructField(field_46,StringType,true), StructField(field_47,IntegerType,true), StructField(field_48,IntegerType,true), StructField(field_49,DateType,true), StructField(field_50,DateType,true), StructField(field_51,DateType,true), StructField(field_52,DateType,true), StructField(field_53,StringType,true), StructField(field_54,StringType,true), StructField(field_55,IntegerType,true), StructField(field_56,DateType,true), StructField(field_57,StringType,true), StructField(field_58,StringType,true), StructField(field_59,StringType,true), StructField(field_60,StringType,true), StructField(field_61,StringType,true), StructField(field_62,StringType,true), StructField(field_63,StringType,true), StructField(field_64,BooleanType,true), StructField(field_65,MapType(StringType,StringType,true),true), StructField(field_66,IntegerType,true), StructField(field_67,IntegerType,true), StructField(field_68,StringType,true), StructField(field_154,StructType(StructField(field_70,LongType,true), StructField(field_71,StringType,true), StructField(field_72,StringType,true), StructField(field_73,LongType,true), StructField(field_74,StringType,true), StructField(field_75,StringType,true), StructField(field_76,StringType,true), StructField(field_77,StringType,true), StructField(field_78,StringType,true), StructField(field_82,StructType(StructField(field_80,StringType,true), StructField(field_81,StringType,true), StructField(field_82,IntegerType,true)),true), StructField(field_86,StructType(StructField(field_84,IntegerType,true), StructField(field_85,IntegerType,true), StructField(field_86,IntegerType,true)),true), StructField(field_87,DoubleType,true), StructField(field_88,DoubleType,true), StructField(field_94,ArrayType(StructType(StructField(field_90,StringType,true), StructField(field_91,StringType,true), StructField(field_92,StringType,true), StructField(field_93,DoubleType,true), StructField(field_94,DoubleType,true)),true),true), StructField(field_108,StructType(StructField(field_96,StringType,true), StructField(field_97,StringType,true), StructField(field_98,StringType,true), StructField(field_99,DoubleType,true), StructField(field_100,StringType,true), StructField(field_101,IntegerType,true), StructField(field_102,StringType,true), StructField(field_103,StringType,true), StructField(field_104,StringType,true), StructField(field_105,IntegerType,true), StructField(field_106,IntegerType,true), StructField(field_107,IntegerType,true), StructField(field_108,IntegerType,true)),true), StructField(field_109,StringType,true), StructField(field_119,StructType(StructField(field_111,StringType,true), StructField(field_112,StringType,true), StructField(field_113,StringType,true), StructField(field_114,StringType,true), StructField(field_115,StringType,true), StructField(field_116,StringType,true), StructField(field_117,StringType,true), StructField(field_118,StringType,true), StructField(field_119,StringType,true)),true), StructField(field_129,StructType(StructField(field_121,StringType,true), StructField(field_122,StringType,true), StructField(field_123,StringType,true), StructField(field_124,StringType,true), StructField(field_125,StringType,true), StructField(field_126,StringType,true), StructField(field_127,StringType,true), StructField(field_128,StringType,true), StructField(field_129,StringType,true)),true), StructField(field_139,StructType(StructField(field_131,StringType,true), StructField(field_132,StringType,true), StructField(field_133,StringType,true), StructField(field_134,StringType,true), StructField(field_135,StringType,true), StructField(field_136,StringType,true), StructField(field_137,StringType,true), StructField(field_138,StringType,true), StructField(field_139,StringType,true)),true), StructField(field_146,ArrayType(StructType(StructField(field_141,LongType,true), StructField(field_142,StringType,true), StructField(field_143,StringType,true), StructField(field_144,StringType,true), StructField(field_145,LongType,true), StructField(field_146,LongType,true)),true),true), StructField(field_147,ArrayType(StringType,true),true), StructField(field_148,ArrayType(StringType,true),true), StructField(field_154,ArrayType(StructType(StructField(field_150,LongType,true), StructField(field_151,StringType,true), StructField(field_152,StringType,true), StructField(field_153,StringType,true), StructField(field_154,ArrayType(StringType,true),true)),true),true)),true), StructField(field_167,StructType(StructField(field_165,StructType(StructField(field_157,DoubleType,true), StructField(field_158,DoubleType,true), StructField(field_159,DoubleType,true), StructField(field_160,DoubleType,true), StructField(field_161,DoubleType,true), StructField(field_162,DecimalType(38,18),true), StructField(field_163,DecimalType(38,18),true), StructField(field_164,DoubleType,true), StructField(field_165,DoubleType,true)),true), StructField(field_166,LongType,true), StructField(field_167,MapType(StringType,StringType,true),true)),true), StructField(field_267,StructType(StructField(field_172,StructType(StructField(field_170,DoubleType,true), StructField(field_171,DoubleType,true), StructField(field_172,StringType,true)),true), StructField(field_191,StructType(StructField(field_174,DoubleType,true), StructField(field_175,DoubleType,true), StructField(field_176,DoubleType,true), StructField(field_177,DoubleType,true), StructField(field_178,DoubleType,true), StructField(field_179,DoubleType,true), StructField(field_180,DoubleType,true), StructField(field_181,DoubleType,true), StructField(field_182,DoubleType,true), StructField(field_183,DoubleType,true), StructField(field_184,DoubleType,true), StructField(field_185,DoubleType,true), StructField(field_186,DoubleType,true), StructField(field_187,IntegerType,true), StructField(field_188,LongType,true), StructField(field_189,LongType,true), StructField(field_190,LongType,true), StructField(field_191,LongType,true)),true), StructField(field_198,StructType(StructField(field_193,DoubleType,true), StructField(field_194,DoubleType,true), StructField(field_195,DoubleType,true), StructField(field_196,DoubleType,true), StructField(field_197,DoubleType,true), StructField(field_198,DoubleType,true)),true), StructField(field_203,StructType(StructField(field_200,DecimalType(38,18),true), StructField(field_201,DoubleType,true), StructField(field_202,DecimalType(38,18),true), StructField(field_203,IntegerType,true)),true), StructField(field_228,StructType(StructField(field_205,DoubleType,true), StructField(field_206,DoubleType,true), StructField(field_207,DoubleType,true), StructField(field_208,DoubleType,true), StructField(field_209,StringType,true), StructField(field_228,StructType(StructField(field_211,IntegerType,true), StructField(field_212,StringType,true), StructField(field_213,DoubleType,true), StructField(field_214,IntegerType,true), StructField(field_215,DoubleType,true), StructField(field_216,IntegerType,true), StructField(field_217,DoubleType,true), StructField(field_218,IntegerType,true), StructField(field_219,IntegerType,true), StructField(field_220,IntegerType,true), StructField(field_221,IntegerType,true), StructField(field_222,StringType,true), StructField(field_223,DateType,true), StructField(field_224,IntegerType,true), StructField(field_225,IntegerType,true), StructField(field_226,IntegerType,true), StructField(field_227,IntegerType,true), StructField(field_228,StringType,true)),true)),true), StructField(field_247,ArrayType(StructType(StructField(field_230,TimestampType,true), StructField(field_231,LongType,true), StructField(field_232,StringType,true), StructField(field_233,StringType,true), StructField(field_234,StringType,true), StructField(field_235,TimestampType,true), StructField(field_236,TimestampType,true), StructField(field_237,DoubleType,true), StructField(field_238,IntegerType,true), StructField(field_239,IntegerType,true), StructField(field_240,BooleanType,true), StructField(field_241,IntegerType,true), StructField(field_242,IntegerType,true), StructField(field_243,DoubleType,true), StructField(field_244,LongType,true), StructField(field_245,StringType,true), StructField(field_246,TimestampType,true), StructField(field_247,StringType,true)),true),true), StructField(field_266,ArrayType(StructType(StructField(field_249,TimestampType,true), StructField(field_250,LongType,true), StructField(field_251,StringType,true), StructField(field_252,StringType,true), StructField(field_253,StringType,true), StructField(field_254,TimestampType,true), StructField(field_255,TimestampType,true), StructField(field_256,DoubleType,true), StructField(field_257,IntegerType,true), StructField(field_258,IntegerType,true), StructField(field_259,BooleanType,true), StructField(field_260,IntegerType,true), StructField(field_261,IntegerType,true), StructField(field_262,DoubleType,true), StructField(field_263,LongType,true), StructField(field_264,StringType,true), StructField(field_265,TimestampType,true), StructField(field_266,StringType,true)),true),true), StructField(field_267,DecimalType(38,18),true)),true), StructField(field_274,ArrayType(StructType(StructField(field_269,StringType,true), StructField(field_270,StringType,true), StructField(field_271,StringType,true), StructField(field_272,StringType,true), StructField(field_273,DoubleType,true), StructField(field_274,StringType,true)),true),true), StructField(field_275,IntegerType,true), StructField(field_290,StructType(StructField(field_277,IntegerType,true), StructField(field_278,BooleanType,true), StructField(field_279,IntegerType,true), StructField(field_280,BooleanType,true), StructField(field_281,BooleanType,true), StructField(field_282,BooleanType,true), StructField(field_283,BooleanType,true), StructField(field_284,BooleanType,true), StructField(field_285,BooleanType,true), StructField(field_286,StringType,true), StructField(field_287,StringType,true), StructField(field_288,StringType,true), StructField(field_289,BooleanType,true), StructField(field_290,BooleanType,true)),true), StructField(field_306,StructType(StructField(field_296,StructType(StructField(field_293,BooleanType,true), StructField(field_294,BooleanType,true), StructField(field_295,StringType,true), StructField(field_296,StringType,true)),true), StructField(field_301,StructType(StructField(field_298,BooleanType,true), StructField(field_299,BooleanType,true), StructField(field_300,StringType,true), StructField(field_301,StringType,true)),true), StructField(field_306,StructType(StructField(field_303,BooleanType,true), StructField(field_304,BooleanType,true), StructField(field_305,StringType,true), StructField(field_306,StringType,true)),true)),true), StructField(field_307,IntegerType,true), StructField(field_308,BooleanType,true), StructField(field_309,BooleanType,true), StructField(field_319,StructType(StructField(field_311,LongType,true), StructField(field_312,LongType,true), StructField(field_313,BooleanType,true), StructField(field_314,IntegerType,true), StructField(field_315,StringType,true), StructField(field_316,StringType,true), StructField(field_317,BooleanType,true), StructField(field_318,BooleanType,true), StructField(field_319,BooleanType,true)),true), StructField(field_320,TimestampType,true), StructField(field_321,DateType,true)),true),true)),true)], obj#53: scala.Tuple2 | |
| +- DeserializeToObject newInstance(class scala.Tuple2), obj#52: scala.Tuple2 | |
| +- Join FullOuter, (_2#37.key1 = _1#36.key1) | |
| :- Project [named_struct(key1, key1#23, value, value#24) AS _1#36] | |
| : +- Project [_1#18 AS key1#23, _2#19 AS value#24] | |
| : +- LocalRelation [_1#18, _2#19] | |
| +- Project [named_struct(key1, key1#7, values, values#8) AS _2#37] | |
| +- Project [_1#2 AS key1#7, _2#3 AS values#8] | |
| +- LocalRelation [_1#2, _2#3] | |
| == Optimized Logical Plan == | |
| SerializeFromObject [if (isnull(unwrapoption(ObjectType(class scala.Tuple2), knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1))) null else named_struct(_1, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class scala.Tuple2), knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1))._1, true, false), _2, knownnotnull(unwrapoption(ObjectType(class scala.Tuple2), knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1))._2) AS _1#54, mapobjects(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1), if (isnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1))) null else named_struct(field_1, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_1), true, false), field_2, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_2), true, false), field_3, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_3), true, false), field_4, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_4), true, false), field_5, staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, TimestampType, fromJavaTimestamp, unwrapoption(ObjectType(class java.sql.Timestamp), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_5), true, false), field_6, staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, DateType, fromJavaDate, unwrapoption(ObjectType(class java.sql.Date), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_6), true, false), field_7, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_7), true, false), field_8, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_8), true, false), field_9, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_9), true, false), field_10, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_10), true, false), field_11, unwrapoption(IntegerType, knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_11), field_12, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_12), true, false), ... 138 more fields), knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2, None) AS _2#55] | |
| +- MapElements ProcessSpec$$Lambda$2764/00000000FB32F620@ad1d95f9, class scala.Tuple2, [StructField(_1,StructType(StructField(_1,StringType,true), StructField(_2,IntegerType,false)),true), StructField(_2,StructType(StructField(_1,StringType,true), StructField(_2,ArrayType(StructType(StructField(field_1,StringType,true), StructField(field_2,StringType,true), StructField(field_3,StringType,true), StructField(field_4,StringType,true), StructField(field_5,TimestampType,true), StructField(field_6,DateType,true), StructField(field_7,StringType,true), StructField(field_8,StringType,true), StructField(field_9,StringType,true), StructField(field_10,StringType,true), StructField(field_11,IntegerType,true), StructField(field_12,StringType,true), StructField(field_13,StringType,true), StructField(field_14,StringType,true), StructField(field_15,StringType,true), StructField(field_16,StringType,true), StructField(field_17,StringType,true), StructField(field_18,DoubleType,true), StructField(field_19,DoubleType,true), StructField(field_20,DoubleType,true), StructField(field_21,DoubleType,true), StructField(field_22,DoubleType,true), StructField(field_23,DoubleType,true), StructField(field_24,DoubleType,true), StructField(field_25,DoubleType,true), StructField(field_26,DoubleType,true), StructField(field_27,DoubleType,true), StructField(field_28,DoubleType,true), StructField(field_29,DoubleType,true), StructField(field_30,StringType,true), StructField(field_31,StringType,true), StructField(field_32,StringType,true), StructField(field_33,StringType,true), StructField(field_34,StringType,true), StructField(field_35,StringType,true), StructField(field_36,StringType,true), StructField(field_37,StringType,true), StructField(field_38,StringType,true), StructField(field_39,StringType,true), StructField(field_40,StringType,true), StructField(field_41,StringType,true), StructField(field_42,StringType,true), StructField(field_43,StringType,true), StructField(field_44,StringType,true), StructField(field_45,StringType,true), StructField(field_46,StringType,true), StructField(field_47,IntegerType,true), StructField(field_48,IntegerType,true), StructField(field_49,DateType,true), StructField(field_50,DateType,true), StructField(field_51,DateType,true), StructField(field_52,DateType,true), StructField(field_53,StringType,true), StructField(field_54,StringType,true), StructField(field_55,IntegerType,true), StructField(field_56,DateType,true), StructField(field_57,StringType,true), StructField(field_58,StringType,true), StructField(field_59,StringType,true), StructField(field_60,StringType,true), StructField(field_61,StringType,true), StructField(field_62,StringType,true), StructField(field_63,StringType,true), StructField(field_64,BooleanType,true), StructField(field_65,MapType(StringType,StringType,true),true), StructField(field_66,IntegerType,true), StructField(field_67,IntegerType,true), StructField(field_68,StringType,true), StructField(field_154,StructType(StructField(field_70,LongType,true), StructField(field_71,StringType,true), StructField(field_72,StringType,true), StructField(field_73,LongType,true), StructField(field_74,StringType,true), StructField(field_75,StringType,true), StructField(field_76,StringType,true), StructField(field_77,StringType,true), StructField(field_78,StringType,true), StructField(field_82,StructType(StructField(field_80,StringType,true), StructField(field_81,StringType,true), StructField(field_82,IntegerType,true)),true), StructField(field_86,StructType(StructField(field_84,IntegerType,true), StructField(field_85,IntegerType,true), StructField(field_86,IntegerType,true)),true), StructField(field_87,DoubleType,true), StructField(field_88,DoubleType,true), StructField(field_94,ArrayType(StructType(StructField(field_90,StringType,true), StructField(field_91,StringType,true), StructField(field_92,StringType,true), StructField(field_93,DoubleType,true), StructField(field_94,DoubleType,true)),true),true), StructField(field_108,StructType(StructField(field_96,StringType,true), StructField(field_97,StringType,true), StructField(field_98,StringType,true), StructField(field_99,DoubleType,true), StructField(field_100,StringType,true), StructField(field_101,IntegerType,true), StructField(field_102,StringType,true), StructField(field_103,StringType,true), StructField(field_104,StringType,true), StructField(field_105,IntegerType,true), StructField(field_106,IntegerType,true), StructField(field_107,IntegerType,true), StructField(field_108,IntegerType,true)),true), StructField(field_109,StringType,true), StructField(field_119,StructType(StructField(field_111,StringType,true), StructField(field_112,StringType,true), StructField(field_113,StringType,true), StructField(field_114,StringType,true), StructField(field_115,StringType,true), StructField(field_116,StringType,true), StructField(field_117,StringType,true), StructField(field_118,StringType,true), StructField(field_119,StringType,true)),true), StructField(field_129,StructType(StructField(field_121,StringType,true), StructField(field_122,StringType,true), StructField(field_123,StringType,true), StructField(field_124,StringType,true), StructField(field_125,StringType,true), StructField(field_126,StringType,true), StructField(field_127,StringType,true), StructField(field_128,StringType,true), StructField(field_129,StringType,true)),true), StructField(field_139,StructType(StructField(field_131,StringType,true), StructField(field_132,StringType,true), StructField(field_133,StringType,true), StructField(field_134,StringType,true), StructField(field_135,StringType,true), StructField(field_136,StringType,true), StructField(field_137,StringType,true), StructField(field_138,StringType,true), StructField(field_139,StringType,true)),true), StructField(field_146,ArrayType(StructType(StructField(field_141,LongType,true), StructField(field_142,StringType,true), StructField(field_143,StringType,true), StructField(field_144,StringType,true), StructField(field_145,LongType,true), StructField(field_146,LongType,true)),true),true), StructField(field_147,ArrayType(StringType,true),true), StructField(field_148,ArrayType(StringType,true),true), StructField(field_154,ArrayType(StructType(StructField(field_150,LongType,true), StructField(field_151,StringType,true), StructField(field_152,StringType,true), StructField(field_153,StringType,true), StructField(field_154,ArrayType(StringType,true),true)),true),true)),true), StructField(field_167,StructType(StructField(field_165,StructType(StructField(field_157,DoubleType,true), StructField(field_158,DoubleType,true), StructField(field_159,DoubleType,true), StructField(field_160,DoubleType,true), StructField(field_161,DoubleType,true), StructField(field_162,DecimalType(38,18),true), StructField(field_163,DecimalType(38,18),true), StructField(field_164,DoubleType,true), StructField(field_165,DoubleType,true)),true), StructField(field_166,LongType,true), StructField(field_167,MapType(StringType,StringType,true),true)),true), StructField(field_267,StructType(StructField(field_172,StructType(StructField(field_170,DoubleType,true), StructField(field_171,DoubleType,true), StructField(field_172,StringType,true)),true), StructField(field_191,StructType(StructField(field_174,DoubleType,true), StructField(field_175,DoubleType,true), StructField(field_176,DoubleType,true), StructField(field_177,DoubleType,true), StructField(field_178,DoubleType,true), StructField(field_179,DoubleType,true), StructField(field_180,DoubleType,true), StructField(field_181,DoubleType,true), StructField(field_182,DoubleType,true), StructField(field_183,DoubleType,true), StructField(field_184,DoubleType,true), StructField(field_185,DoubleType,true), StructField(field_186,DoubleType,true), StructField(field_187,IntegerType,true), StructField(field_188,LongType,true), StructField(field_189,LongType,true), StructField(field_190,LongType,true), StructField(field_191,LongType,true)),true), StructField(field_198,StructType(StructField(field_193,DoubleType,true), StructField(field_194,DoubleType,true), StructField(field_195,DoubleType,true), StructField(field_196,DoubleType,true), StructField(field_197,DoubleType,true), StructField(field_198,DoubleType,true)),true), StructField(field_203,StructType(StructField(field_200,DecimalType(38,18),true), StructField(field_201,DoubleType,true), StructField(field_202,DecimalType(38,18),true), StructField(field_203,IntegerType,true)),true), StructField(field_228,StructType(StructField(field_205,DoubleType,true), StructField(field_206,DoubleType,true), StructField(field_207,DoubleType,true), StructField(field_208,DoubleType,true), StructField(field_209,StringType,true), StructField(field_228,StructType(StructField(field_211,IntegerType,true), StructField(field_212,StringType,true), StructField(field_213,DoubleType,true), StructField(field_214,IntegerType,true), StructField(field_215,DoubleType,true), StructField(field_216,IntegerType,true), StructField(field_217,DoubleType,true), StructField(field_218,IntegerType,true), StructField(field_219,IntegerType,true), StructField(field_220,IntegerType,true), StructField(field_221,IntegerType,true), StructField(field_222,StringType,true), StructField(field_223,DateType,true), StructField(field_224,IntegerType,true), StructField(field_225,IntegerType,true), StructField(field_226,IntegerType,true), StructField(field_227,IntegerType,true), StructField(field_228,StringType,true)),true)),true), StructField(field_247,ArrayType(StructType(StructField(field_230,TimestampType,true), StructField(field_231,LongType,true), StructField(field_232,StringType,true), StructField(field_233,StringType,true), StructField(field_234,StringType,true), StructField(field_235,TimestampType,true), StructField(field_236,TimestampType,true), StructField(field_237,DoubleType,true), StructField(field_238,IntegerType,true), StructField(field_239,IntegerType,true), StructField(field_240,BooleanType,true), StructField(field_241,IntegerType,true), StructField(field_242,IntegerType,true), StructField(field_243,DoubleType,true), StructField(field_244,LongType,true), StructField(field_245,StringType,true), StructField(field_246,TimestampType,true), StructField(field_247,StringType,true)),true),true), StructField(field_266,ArrayType(StructType(StructField(field_249,TimestampType,true), StructField(field_250,LongType,true), StructField(field_251,StringType,true), StructField(field_252,StringType,true), StructField(field_253,StringType,true), StructField(field_254,TimestampType,true), StructField(field_255,TimestampType,true), StructField(field_256,DoubleType,true), StructField(field_257,IntegerType,true), StructField(field_258,IntegerType,true), StructField(field_259,BooleanType,true), StructField(field_260,IntegerType,true), StructField(field_261,IntegerType,true), StructField(field_262,DoubleType,true), StructField(field_263,LongType,true), StructField(field_264,StringType,true), StructField(field_265,TimestampType,true), StructField(field_266,StringType,true)),true),true), StructField(field_267,DecimalType(38,18),true)),true), StructField(field_274,ArrayType(StructType(StructField(field_269,StringType,true), StructField(field_270,StringType,true), StructField(field_271,StringType,true), StructField(field_272,StringType,true), StructField(field_273,DoubleType,true), StructField(field_274,StringType,true)),true),true), StructField(field_275,IntegerType,true), StructField(field_290,StructType(StructField(field_277,IntegerType,true), StructField(field_278,BooleanType,true), StructField(field_279,IntegerType,true), StructField(field_280,BooleanType,true), StructField(field_281,BooleanType,true), StructField(field_282,BooleanType,true), StructField(field_283,BooleanType,true), StructField(field_284,BooleanType,true), StructField(field_285,BooleanType,true), StructField(field_286,StringType,true), StructField(field_287,StringType,true), StructField(field_288,StringType,true), StructField(field_289,BooleanType,true), StructField(field_290,BooleanType,true)),true), StructField(field_306,StructType(StructField(field_296,StructType(StructField(field_293,BooleanType,true), StructField(field_294,BooleanType,true), StructField(field_295,StringType,true), StructField(field_296,StringType,true)),true), StructField(field_301,StructType(StructField(field_298,BooleanType,true), StructField(field_299,BooleanType,true), StructField(field_300,StringType,true), StructField(field_301,StringType,true)),true), StructField(field_306,StructType(StructField(field_303,BooleanType,true), StructField(field_304,BooleanType,true), StructField(field_305,StringType,true), StructField(field_306,StringType,true)),true)),true), StructField(field_307,IntegerType,true), StructField(field_308,BooleanType,true), StructField(field_309,BooleanType,true), StructField(field_319,StructType(StructField(field_311,LongType,true), StructField(field_312,LongType,true), StructField(field_313,BooleanType,true), StructField(field_314,IntegerType,true), StructField(field_315,StringType,true), StructField(field_316,StringType,true), StructField(field_317,BooleanType,true), StructField(field_318,BooleanType,true), StructField(field_319,BooleanType,true)),true), StructField(field_320,TimestampType,true), StructField(field_321,DateType,true)),true),true)),true)], obj#53: scala.Tuple2 | |
| +- DeserializeToObject newInstance(class scala.Tuple2), obj#52: scala.Tuple2 | |
| +- Join FullOuter, (_2#37.key1 = _1#36.key1) | |
| :- LocalRelation [_1#36] | |
| +- LocalRelation [_2#37] | |
| == Physical Plan == | |
| SerializeFromObject [if (isnull(unwrapoption(ObjectType(class scala.Tuple2), knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1))) null else named_struct(_1, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class scala.Tuple2), knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1))._1, true, false), _2, knownnotnull(unwrapoption(ObjectType(class scala.Tuple2), knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1))._2) AS _1#54, mapobjects(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1), if (isnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1))) null else named_struct(field_1, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_1), true, false), field_2, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_2), true, false), field_3, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_3), true, false), field_4, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_4), true, false), field_5, staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, TimestampType, fromJavaTimestamp, unwrapoption(ObjectType(class java.sql.Timestamp), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_5), true, false), field_6, staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, DateType, fromJavaDate, unwrapoption(ObjectType(class java.sql.Date), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_6), true, false), field_7, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_7), true, false), field_8, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_8), true, false), field_9, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_9), true, false), field_10, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_10), true, false), field_11, unwrapoption(IntegerType, knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_11), field_12, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(lambdavariable(MapObject, ObjectType(class MyBigDomainClass), true, -1)).field_12), true, false), ... 138 more fields), knownnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2, None) AS _2#55] | |
| +- MapElements ProcessSpec$$Lambda$2764/00000000FB32F620@ad1d95f9, obj#53: scala.Tuple2 | |
| +- DeserializeToObject newInstance(class scala.Tuple2), obj#52: scala.Tuple2 | |
| +- SortMergeJoin [_1#36.key1], [_2#37.key1], FullOuter | |
| :- *(1) Sort [_1#36.key1 ASC NULLS FIRST], false, 0 | |
| : +- Exchange hashpartitioning(_1#36.key1, 200), ENSURE_REQUIREMENTS, [id=#77] | |
| : +- LocalTableScan [_1#36] | |
| +- Sort [_2#37.key1 ASC NULLS FIRST], false, 0 | |
| +- Exchange hashpartitioning(_2#37.key1, 200), ENSURE_REQUIREMENTS, [id=#78] | |
| +- LocalTableScan [_2#37] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment