Skip to content

Instantly share code, notes, and snippets.

@rubensayshi
Created April 25, 2012 07:13
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save rubensayshi/2487666 to your computer and use it in GitHub Desktop.
Save rubensayshi/2487666 to your computer and use it in GitHub Desktop.
error when doing lateral view + join in subquery
-- this is the query that causes the crash
SELECT
date_int,
gender,
SUM( `generic`['count'] ) AS count
FROM (
SELECT
memberId,
date_int,
members_map.gender AS gender,
generic
FROM visit_stats
JOIN members_map ON(members_map.member_id = visit_stats.memberId)
) visit_stats
GROUP BY date_int, gender
CREATE TABLE members_map (
member_id INT,
gender INT,
birthdate STRING
)
ROW FORMAT
DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE
CREATE TABLE visit_stats (
`date` STRING,
`memberId` INT,
`generic` MAP<STRING, INT>,
`platforms` MAP<STRING, MAP<STRING, INT>>,
`parts` MAP<STRING, MAP<STRING, INT>>,
`devices_by_platform` MAP<STRING, MAP<STRING, MAP<STRING, INT>>>
)
PARTITIONED BY (date_int INT)
ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
STORED AS SEQUENCEFILE
java.lang.RuntimeException: Error in configuring object
at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93)
at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64)
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117)
at org.apache.hadoop.mapred.ReduceTask.runOldReducer(ReduceTask.java:431)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:416)
at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115)
at org.apache.hadoop.mapred.Child.main(Child.java:262)
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
... 9 more
Caused by: java.lang.RuntimeException: Reduce operator initialization failed
at org.apache.hadoop.hive.ql.exec.ExecReducer.configure(ExecReducer.java:157)
... 14 more
Caused by: java.lang.RuntimeException: cannot find field _col1 from [0:_col4, 1:_col6, 2:_col10]
at org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.getStandardStructFieldRef(ObjectInspectorUtils.java:345)
at org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector.getStructFieldRef(StandardStructObjectInspector.java:143)
at org.apache.hadoop.hive.ql.exec.ExprNodeColumnEvaluator.initialize(ExprNodeColumnEvaluator.java:57)
at org.apache.hadoop.hive.ql.exec.Operator.initEvaluators(Operator.java:896)
at org.apache.hadoop.hive.ql.exec.Operator.initEvaluatorsAndReturnStruct(Operator.java:922)
at org.apache.hadoop.hive.ql.exec.SelectOperator.initializeOp(SelectOperator.java:60)
at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:357)
at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:433)
at org.apache.hadoop.hive.ql.exec.Operator.initializeChildren(Operator.java:389)
at org.apache.hadoop.hive.ql.exec.JoinOperator.initializeOp(JoinOperator.java:60)
at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:357)
at org.apache.hadoop.hive.ql.exec.ExecReducer.configure(ExecReducer.java:150)
... 14 more
-- the same query as the crashing one
-- except I've added the COUNT ()
SELECT
date_int,
gender,
COUNT( memberId ) AS unique,
SUM( `generic`['count'] ) AS count
FROM (
SELECT
memberId,
date_int,
members_map.gender AS gender,
generic
FROM visit_stats
JOIN members_map ON(members_map.member_id = visit_stats.memberId)
) visit_stats
GROUP BY date_int, gender
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment