Skip to content

Instantly share code, notes, and snippets.

@rubensayshi

rubensayshi/plan.xml

Created April 26, 2012 13:36
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save rubensayshi/2499658 to your computer and use it in GitHub Desktop.
Save rubensayshi/2499658 to your computer and use it in GitHub Desktop.
Download ZIP
Hive MAPJOIN + LATERAL VIEW
Raw
plan.xml
<?xml version="1.0" encoding="UTF-8"?>
<java version="1.6.0_26" class="java.beans.XMLDecoder">
<object class="org.apache.hadoop.hive.ql.plan.MapredLocalWork">
<void property="aliasToFetchWork">
<object class="java.util.LinkedHashMap">
<void method="put">
<string>visit_stats:members_map</string>
<object class="org.apache.hadoop.hive.ql.plan.FetchWork">
<void property="limit">
<int>-1</int>
</void>
<void property="tblDesc">
<object class="org.apache.hadoop.hive.ql.plan.TableDesc">
<void property="deserializerClass">
<class>org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe</class>
</void>
<void property="inputFileFormatClass">
<class>org.apache.hadoop.mapred.TextInputFormat</class>
</void>
<void property="outputFileFormatClass">
<class>org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat</class>
</void>
<void property="properties">
<object class="java.util.Properties">
<void method="put">
<string>name</string>
<string>hive_mapjoin.members_map</string>
</void>
<void method="put">
<string>field.delim</string>
<string>,</string>
</void>
<void method="put">
<string>columns.types</string>
<string>int:int:string</string>
</void>
<void method="put">
<string>serialization.ddl</string>
<string>struct members_map { i32 member_id, i32 gender, string birthdate}</string>
</void>
<void method="put">
<string>columns</string>
<string>member_id,gender,birthdate</string>
</void>
<void method="put">
<string>serialization.format</string>
<string>,</string>
</void>
<void method="put">
<string>bucket_count</string>
<string>-1</string>
</void>
<void method="put">
<string>serialization.lib</string>
<string>org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe</string>
</void>
<void method="put">
<string>file.inputformat</string>
<string>org.apache.hadoop.mapred.TextInputFormat</string>
</void>
<void method="put">
<string>file.outputformat</string>
<string>org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat</string>
</void>
<void method="put">
<string>location</string>
<string>hdfs://localhost:54310/user/hive/warehouse/hive_mapjoin.db/members_map</string>
</void>
<void method="put">
<string>transient_lastDdlTime</string>
<string>1335445506</string>
</void>
</object>
</void>
<void property="serdeClassName">
<string>org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe</string>
</void>
</object>
</void>
<void property="tblDir">
<string>hdfs://localhost:54310/user/hive/warehouse/hive_mapjoin.db/members_map</string>
</void>
</object>
</void>
</object>
</void>
<void property="aliasToWork">
<object class="java.util.LinkedHashMap">
<void method="put">
<string>visit_stats:members_map</string>
<object id="TableScanOperator0" class="org.apache.hadoop.hive.ql.exec.TableScanOperator">
<void property="childOperators">
<object class="java.util.ArrayList">
<void method="add">
<object class="org.apache.hadoop.hive.ql.exec.HashTableSinkOperator">
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.HashTableSinkDesc">
<void property="conds">
<array class="org.apache.hadoop.hive.ql.plan.JoinCondDesc" length="1">
<void index="0">
<object class="org.apache.hadoop.hive.ql.plan.JoinCondDesc">
<void property="right">
<int>1</int>
</void>
</object>
</void>
</array>
</void>
<void property="dumpFilePrefix">
<string>members_map-0</string>
</void>
<void property="exprs">
<object class="java.util.HashMap">
<void method="put">
<byte>0</byte>
<object class="java.util.ArrayList">
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc">
<void property="column">
<string>date_int</string>
</void>
<void property="tabAlias">
<string>visit_stats</string>
</void>
<void property="typeInfo">
<object id="PrimitiveTypeInfo0" class="org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo">
<void property="typeName">
<string>int</string>
</void>
</object>
</void>
</object>
</void>
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc">
<void property="column">
<string>member_id</string>
</void>
<void property="tabAlias">
<string>visit_stats</string>
</void>
<void property="typeInfo">
<object idref="PrimitiveTypeInfo0"/>
</void>
</object>
</void>
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc">
<void property="column">
<string>parts</string>
</void>
<void property="tabAlias">
<string>visit_stats</string>
</void>
<void property="typeInfo">
<object class="org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo">
<void property="mapKeyTypeInfo">
<object id="PrimitiveTypeInfo1" class="org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo">
<void property="typeName">
<string>string</string>
</void>
</object>
</void>
<void property="mapValueTypeInfo">
<object class="org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo">
<void property="mapKeyTypeInfo">
<object idref="PrimitiveTypeInfo1"/>
</void>
<void property="mapValueTypeInfo">
<object idref="PrimitiveTypeInfo0"/>
</void>
</object>
</void>
</object>
</void>
</object>
</void>
</object>
</void>
<void method="put">
<byte>1</byte>
<object class="java.util.ArrayList">
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc">
<void property="column">
<string>gender</string>
</void>
<void property="tabAlias">
<string>members_map</string>
</void>
<void property="typeInfo">
<object idref="PrimitiveTypeInfo0"/>
</void>
</object>
</void>
</object>
</void>
</object>
</void>
<void property="filters">
<object class="java.util.HashMap">
<void method="put">
<byte>0</byte>
<object class="java.util.ArrayList"/>
</void>
<void method="put">
<byte>1</byte>
<object class="java.util.ArrayList"/>
</void>
</object>
</void>
<void property="hashtableMemoryUsage">
<float>0.9</float>
</void>
<void property="keyTblDesc">
<object class="org.apache.hadoop.hive.ql.plan.TableDesc">
<void property="deserializerClass">
<class>org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe</class>
</void>
<void property="inputFileFormatClass">
<class>org.apache.hadoop.mapred.SequenceFileInputFormat</class>
</void>
<void property="outputFileFormatClass">
<class>org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat</class>
</void>
<void property="properties">
<object class="java.util.Properties">
<void method="put">
<string>columns</string>
<string>mapjoinkey0</string>
</void>
<void method="put">
<string>columns.types</string>
<string>int</string>
</void>
<void method="put">
<string>escape.delim</string>
<string>\</string>
</void>
</object>
</void>
</object>
</void>
<void property="keys">
<object class="java.util.HashMap">
<void method="put">
<byte>0</byte>
<object class="java.util.ArrayList">
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc">
<void property="column">
<string>member_id</string>
</void>
<void property="tabAlias">
<string>visit_stats</string>
</void>
<void property="typeInfo">
<object idref="PrimitiveTypeInfo0"/>
</void>
</object>
</void>
</object>
</void>
<void method="put">
<byte>1</byte>
<object class="java.util.ArrayList">
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc">
<void property="column">
<string>member_id</string>
</void>
<void property="tabAlias">
<string>members_map</string>
</void>
<void property="typeInfo">
<object idref="PrimitiveTypeInfo0"/>
</void>
</object>
</void>
</object>
</void>
</object>
</void>
<void property="noOuterJoin">
<boolean>true</boolean>
</void>
<void property="outputColumnNames">
<object class="java.util.ArrayList">
<void method="add">
<string>_col0</string>
</void>
<void method="add">
<string>_col1</string>
</void>
<void method="add">
<string>_col2</string>
</void>
<void method="add">
<string>_col6</string>
</void>
</object>
</void>
<void property="retainList">
<object class="java.util.HashMap">
<void method="put">
<byte>0</byte>
<object class="java.util.ArrayList">
<void method="add">
<int>0</int>
</void>
<void method="add">
<int>1</int>
</void>
<void method="add">
<int>2</int>
</void>
</object>
</void>
<void method="put">
<byte>1</byte>
<object class="java.util.ArrayList">
<void method="add">
<int>0</int>
</void>
</object>
</void>
</object>
</void>
<void property="reversedExprs">
<object class="java.util.HashMap">
<void method="put">
<string>_col8</string>
<byte>1</byte>
</void>
<void method="put">
<string>_col7</string>
<byte>1</byte>
</void>
<void method="put">
<string>_col6</string>
<byte>1</byte>
</void>
<void method="put">
<string>_col5</string>
<byte>1</byte>
</void>
<void method="put">
<string>_col4</string>
<byte>0</byte>
</void>
<void method="put">
<string>_col3</string>
<byte>0</byte>
</void>
<void method="put">
<string>_col2</string>
<byte>0</byte>
</void>
<void method="put">
<string>_col1</string>
<byte>0</byte>
</void>
<void method="put">
<string>_col0</string>
<byte>0</byte>
</void>
<void method="put">
<string>_col9</string>
<byte>1</byte>
</void>
</object>
</void>
<void property="tagOrder">
<array class="java.lang.Byte" length="2">
<void index="0">
<byte>0</byte>
</void>
<void index="1">
<byte>1</byte>
</void>
</array>
</void>
<void property="valueTblDescs">
<object class="java.util.ArrayList">
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.TableDesc">
<void property="deserializerClass">
<class>org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe</class>
</void>
<void property="inputFileFormatClass">
<class>org.apache.hadoop.mapred.SequenceFileInputFormat</class>
</void>
<void property="outputFileFormatClass">
<class>org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat</class>
</void>
<void property="properties">
<object class="java.util.Properties">
<void method="put">
<string>columns</string>
<string>mapjoinvalue0,mapjoinvalue1,mapjoinvalue2</string>
</void>
<void method="put">
<string>columns.types</string>
<string>int,int,map&lt;string,map&lt;string,int&gt;&gt;</string>
</void>
<void method="put">
<string>escape.delim</string>
<string>\</string>
</void>
</object>
</void>
</object>
</void>
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.TableDesc">
<void property="deserializerClass">
<class>org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe</class>
</void>
<void property="inputFileFormatClass">
<class>org.apache.hadoop.mapred.SequenceFileInputFormat</class>
</void>
<void property="outputFileFormatClass">
<class>org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat</class>
</void>
<void property="properties">
<object class="java.util.Properties">
<void method="put">
<string>columns</string>
<string>mapjoinvalue0</string>
</void>
<void method="put">
<string>columns.types</string>
<string>int</string>
</void>
<void method="put">
<string>escape.delim</string>
<string>\</string>
</void>
</object>
</void>
</object>
</void>
</object>
</void>
<void property="valueTblFilteredDescs">
<object class="java.util.ArrayList">
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.TableDesc">
<void property="deserializerClass">
<class>org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe</class>
</void>
<void property="inputFileFormatClass">
<class>org.apache.hadoop.mapred.SequenceFileInputFormat</class>
</void>
<void property="outputFileFormatClass">
<class>org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat</class>
</void>
<void property="properties">
<object class="java.util.Properties">
<void method="put">
<string>columns</string>
<string>mapjoinvalue0,mapjoinvalue1,mapjoinvalue2</string>
</void>
<void method="put">
<string>columns.types</string>
<string>int,int,map&lt;string,map&lt;string,int&gt;&gt;</string>
</void>
<void method="put">
<string>escape.delim</string>
<string>\</string>
</void>
</object>
</void>
</object>
</void>
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.TableDesc">
<void property="deserializerClass">
<class>org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe</class>
</void>
<void property="inputFileFormatClass">
<class>org.apache.hadoop.mapred.SequenceFileInputFormat</class>
</void>
<void property="outputFileFormatClass">
<class>org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat</class>
</void>
<void property="properties">
<object class="java.util.Properties">
<void method="put">
<string>columns</string>
<string>mapjoinvalue0</string>
</void>
<void method="put">
<string>columns.types</string>
<string>int</string>
</void>
<void method="put">
<string>escape.delim</string>
<string>\</string>
</void>
</object>
</void>
</object>
</void>
</object>
</void>
</object>
</void>
<void property="counterNames">
<object class="java.util.ArrayList">
<void method="add">
<string>CNTR_NAME_HASHTABLESINK_20_NUM_INPUT_ROWS</string>
</void>
<void method="add">
<string>CNTR_NAME_HASHTABLESINK_20_NUM_OUTPUT_ROWS</string>
</void>
<void method="add">
<string>CNTR_NAME_HASHTABLESINK_20_TIME_TAKEN</string>
</void>
<void method="add">
<string>CNTR_NAME_HASHTABLESINK_20_FATAL_ERROR</string>
</void>
</object>
</void>
<void property="operatorId">
<string>HASHTABLESINK_20</string>
</void>
<void property="parentOperators">
<object class="java.util.ArrayList">
<void method="add">
<null/>
</void>
<void method="add">
<object idref="TableScanOperator0"/>
</void>
</object>
</void>
</object>
</void>
</object>
</void>
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.TableScanDesc">
<void property="alias">
<string>members_map</string>
</void>
<void property="virtualCols">
<object class="java.util.ArrayList"/>
</void>
</object>
</void>
<void property="counterNames">
<object class="java.util.ArrayList">
<void method="add">
<string>CNTR_NAME_TS_1_NUM_INPUT_ROWS</string>
</void>
<void method="add">
<string>CNTR_NAME_TS_1_NUM_OUTPUT_ROWS</string>
</void>
<void method="add">
<string>CNTR_NAME_TS_1_TIME_TAKEN</string>
</void>
<void method="add">
<string>CNTR_NAME_TS_1_FATAL_ERROR</string>
</void>
</object>
</void>
<void property="neededColumnIDs">
<object class="java.util.ArrayList">
<void method="add">
<int>1</int>
</void>
<void method="add">
<int>0</int>
</void>
</object>
</void>
<void property="operatorId">
<string>TS_1</string>
</void>
<void property="schema">
<object class="org.apache.hadoop.hive.ql.exec.RowSchema">
<void property="signature">
<object class="java.util.ArrayList">
<void method="add">
<object class="org.apache.hadoop.hive.ql.exec.ColumnInfo">
<void property="internalName">
<string>member_id</string>
</void>
<void property="tabAlias">
<string>members_map</string>
</void>
<void property="type">
<object idref="PrimitiveTypeInfo0"/>
</void>
</object>
</void>
<void method="add">
<object class="org.apache.hadoop.hive.ql.exec.ColumnInfo">
<void property="internalName">
<string>gender</string>
</void>
<void property="tabAlias">
<string>members_map</string>
</void>
<void property="type">
<object idref="PrimitiveTypeInfo0"/>
</void>
</object>
</void>
<void method="add">
<object class="org.apache.hadoop.hive.ql.exec.ColumnInfo">
<void property="internalName">
<string>birthdate</string>
</void>
<void property="tabAlias">
<string>members_map</string>
</void>
<void property="type">
<object idref="PrimitiveTypeInfo1"/>
</void>
</object>
</void>
<void method="add">
<object class="org.apache.hadoop.hive.ql.exec.ColumnInfo">
<void property="hiddenVirtualCol">
<boolean>true</boolean>
</void>
<void property="internalName">
<string>BLOCK__OFFSET__INSIDE__FILE</string>
</void>
<void property="tabAlias">
<string>members_map</string>
</void>
<void property="type">
<object class="org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo">
<void property="typeName">
<string>bigint</string>
</void>
</object>
</void>
</object>
</void>
<void method="add">
<object class="org.apache.hadoop.hive.ql.exec.ColumnInfo">
<void property="hiddenVirtualCol">
<boolean>true</boolean>
</void>
<void property="internalName">
<string>INPUT__FILE__NAME</string>
</void>
<void property="tabAlias">
<string>members_map</string>
</void>
<void property="type">
<object idref="PrimitiveTypeInfo1"/>
</void>
</object>
</void>
</object>
</void>
</object>
</void>
</object>
</void>
</object>
</void>
<void property="tmpFileURI">
<string>file:/tmp/hduser/hive_2012-04-26_15-35-10_035_304767365355903858/-local-10003/HashTable-Stage-1</string>
</void>
</object>
</java>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment