Created
April 25, 2012 07:13
-
-
Save rubensayshi/2487666 to your computer and use it in GitHub Desktop.
error when doing lateral view + join in subquery
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
-- this is the query that causes the crash | |
SELECT | |
date_int, | |
gender, | |
SUM( `generic`['count'] ) AS count | |
FROM ( | |
SELECT | |
memberId, | |
date_int, | |
members_map.gender AS gender, | |
generic | |
FROM visit_stats | |
JOIN members_map ON(members_map.member_id = visit_stats.memberId) | |
) visit_stats | |
GROUP BY date_int, gender |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
CREATE TABLE members_map ( | |
member_id INT, | |
gender INT, | |
birthdate STRING | |
) | |
ROW FORMAT | |
DELIMITED FIELDS TERMINATED BY ',' | |
STORED AS TEXTFILE |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
CREATE TABLE visit_stats ( | |
`date` STRING, | |
`memberId` INT, | |
`generic` MAP<STRING, INT>, | |
`platforms` MAP<STRING, MAP<STRING, INT>>, | |
`parts` MAP<STRING, MAP<STRING, INT>>, | |
`devices_by_platform` MAP<STRING, MAP<STRING, MAP<STRING, INT>>> | |
) | |
PARTITIONED BY (date_int INT) | |
ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe' | |
STORED AS SEQUENCEFILE |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
java.lang.RuntimeException: Error in configuring object | |
at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93) | |
at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64) | |
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117) | |
at org.apache.hadoop.mapred.ReduceTask.runOldReducer(ReduceTask.java:431) | |
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:416) | |
at org.apache.hadoop.mapred.Child$4.run(Child.java:268) | |
at java.security.AccessController.doPrivileged(Native Method) | |
at javax.security.auth.Subject.doAs(Subject.java:396) | |
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115) | |
at org.apache.hadoop.mapred.Child.main(Child.java:262) | |
Caused by: java.lang.reflect.InvocationTargetException | |
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) | |
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) | |
at java.lang.reflect.Method.invoke(Method.java:597) | |
at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88) | |
... 9 more | |
Caused by: java.lang.RuntimeException: Reduce operator initialization failed | |
at org.apache.hadoop.hive.ql.exec.ExecReducer.configure(ExecReducer.java:157) | |
... 14 more | |
Caused by: java.lang.RuntimeException: cannot find field _col1 from [0:_col4, 1:_col6, 2:_col10] | |
at org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.getStandardStructFieldRef(ObjectInspectorUtils.java:345) | |
at org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector.getStructFieldRef(StandardStructObjectInspector.java:143) | |
at org.apache.hadoop.hive.ql.exec.ExprNodeColumnEvaluator.initialize(ExprNodeColumnEvaluator.java:57) | |
at org.apache.hadoop.hive.ql.exec.Operator.initEvaluators(Operator.java:896) | |
at org.apache.hadoop.hive.ql.exec.Operator.initEvaluatorsAndReturnStruct(Operator.java:922) | |
at org.apache.hadoop.hive.ql.exec.SelectOperator.initializeOp(SelectOperator.java:60) | |
at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:357) | |
at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:433) | |
at org.apache.hadoop.hive.ql.exec.Operator.initializeChildren(Operator.java:389) | |
at org.apache.hadoop.hive.ql.exec.JoinOperator.initializeOp(JoinOperator.java:60) | |
at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:357) | |
at org.apache.hadoop.hive.ql.exec.ExecReducer.configure(ExecReducer.java:150) | |
... 14 more |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
-- the same query as the crashing one | |
-- except I've added the COUNT () | |
SELECT | |
date_int, | |
gender, | |
COUNT( memberId ) AS unique, | |
SUM( `generic`['count'] ) AS count | |
FROM ( | |
SELECT | |
memberId, | |
date_int, | |
members_map.gender AS gender, | |
generic | |
FROM visit_stats | |
JOIN members_map ON(members_map.member_id = visit_stats.memberId) | |
) visit_stats | |
GROUP BY date_int, gender |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment