Skip to content

Instantly share code, notes, and snippets.

@hellojinjie

hellojinjie/GenericUDAFCdnBytesLoaded.java

Created September 29, 2013 08:46
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save hellojinjie/6750572 to your computer and use it in GitHub Desktop.
Save hellojinjie/6750572 to your computer and use it in GitHub Desktop.
Download ZIP
NullPointException in UDAF
Raw
GenericUDAFCdnBytesLoaded.java
package com.jaunty.hive.udf;
import java.util.Map;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFParameterInfo;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver2;
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import com.google.common.collect.Maps;
@Description(value="_FUNC_ calculate the cdn bytes loaded", name="cdnbytesloaded")
public class GenericUDAFCdnBytesLoaded implements GenericUDAFResolver2 {
@Override
public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters)
throws SemanticException {
throw new SemanticException(
"This UDAF does not support the deprecated getEvaluator() method.");
}
@Override
public GenericUDAFEvaluator getEvaluator(GenericUDAFParameterInfo info)
throws SemanticException {
ObjectInspector[] ois = info.getParameterObjectInspectors();
if (ois.length != 1) {
throw new UDFArgumentLengthException("_FUNC_ take only one parameter");
}
return new GenericUDAFCdnBytesLoadedEvaluator();
}
public static class CdnAggregationBuffer implements AggregationBuffer {
Map<String, Long> bytesLoaded;
}
public static class GenericUDAFCdnBytesLoadedEvaluator extends GenericUDAFEvaluator {
private MapObjectInspector inputOI;
private MapObjectInspector outputOI;
private Map<String, String> cdnPatterns;
{
cdnPatterns = Maps.newHashMap();
cdnPatterns.put("cdnl3nl", "Level3");
cdnPatterns.put("cdnak", "Akamai");
cdnPatterns.put("cndllnwnl", "LimeLight");
cdnPatterns.put("cdncd", "CDNetworks");
}
@Override
public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException {
super.init(m, parameters);
if (parameters.length != 1) {
throw new UDFArgumentLengthException("In GenericUDAFCdnBytesLoadedEvaluator "
+ "parameters length is not 1");
}
inputOI = (MapObjectInspector) parameters[0];
outputOI = (MapObjectInspector) ObjectInspectorUtils
.getStandardObjectInspector(inputOI, ObjectInspectorCopyOption.JAVA);
return outputOI;
}
@Override
public AggregationBuffer getNewAggregationBuffer() throws HiveException {
CdnAggregationBuffer buffer = new CdnAggregationBuffer();
buffer.bytesLoaded = Maps.newHashMap();
return buffer;
}
@Override
public void reset(AggregationBuffer agg) throws HiveException {
CdnAggregationBuffer buffer = (CdnAggregationBuffer) agg;
buffer.bytesLoaded.clear();
}
@Override
public void iterate(AggregationBuffer agg, Object[] parameters)
throws HiveException {
if (parameters.length == 1 && parameters[0] != null) {
merge(agg, parameters[0]);
}
}
@Override
public Object terminatePartial(AggregationBuffer agg) throws HiveException {
return ((CdnAggregationBuffer) agg).bytesLoaded;
}
@Override
public void merge(AggregationBuffer agg, Object partial) throws HiveException {
if (partial == null) {
return;
}
try {
CdnAggregationBuffer buffer = (CdnAggregationBuffer) agg;
@SuppressWarnings("unchecked")
Map<String, Long> bytesLoaded = (Map<String, Long>) inputOI.getMap(partial);
for (String key : bytesLoaded.keySet()) {
boolean matched = false;
for (String pattern : cdnPatterns.keySet()) {
if (key.contains(pattern)) {
String cdnName = cdnPatterns.get(pattern);
Long loaded = buffer.bytesLoaded.get(cdnName);
if (loaded == null) {
loaded = bytesLoaded.get(key);
} else {
loaded = loaded > bytesLoaded.get(key) ? loaded : bytesLoaded.get(key);
}
buffer.bytesLoaded.put(cdnName, loaded);
matched = true;
break;
}
}
if (matched == false) {
Long loaded = buffer.bytesLoaded.get(key);
if (loaded == null) {
loaded = bytesLoaded.get(key);
} else {
loaded = loaded > bytesLoaded.get(key) ? loaded : bytesLoaded.get(key);
}
buffer.bytesLoaded.put(key, loaded);
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
public Object terminate(AggregationBuffer agg) throws HiveException {
return ((CdnAggregationBuffer) agg).bytesLoaded;
}
}
}
Raw
task.log
Task Logs: 'attempt_201309271638_0025_m_000000_0'
stdout logs
stderr logs
java.lang.reflect.InvocationTargetException
Continuing ...
java.lang.IllegalArgumentException: Unbound variable: GenericUDAFCdnBytesLoaded$GenericUDAFCdnBytesLoadedEvaluator0
Continuing ...
syslog logs
2013-09-29 16:41:13,328 INFO org.apache.hadoop.util.NativeCodeLoader: Loaded the native-hadoop library
2013-09-29 16:41:13,509 INFO org.apache.hadoop.mapred.TaskRunner: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/distcache/6000556044884442336_-388038660_1759672009/172.16.0.237/tmp/hive-qos/hive_2013-09-29_16-42-52_508_3403558341003759013/-mr-10003/41af96bb-2da6-4a3d-b17c-6d99530c7239 <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/HIVE_PLAN41af96bb-2da6-4a3d-b17c-6d99530c7239
2013-09-29 16:41:13,514 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/META-INF <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/META-INF
2013-09-29 16:41:13,515 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/hive-exec-log4j.properties <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/hive-exec-log4j.properties
2013-09-29 16:41:13,516 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/job.jar <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/job.jar
2013-09-29 16:41:13,517 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/org <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/org
2013-09-29 16:41:13,517 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/javolution <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/javolution
2013-09-29 16:41:13,518 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/javax <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/javax
2013-09-29 16:41:13,519 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/.job.jar.crc <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/.job.jar.crc
2013-09-29 16:41:13,520 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Creating symlink: /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/javaewah <- /home/qos/app/hadoop/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/javaewah
2013-09-29 16:41:13,811 INFO org.apache.hadoop.util.ProcessTree: setsid exited with exit code 0
2013-09-29 16:41:13,815 INFO org.apache.hadoop.mapred.Task: Using ResourceCalculatorPlugin : org.apache.hadoop.util.LinuxResourceCalculatorPlugin@5266f887
2013-09-29 16:41:13,923 INFO org.apache.hadoop.mapred.MapTask: Processing split: Paths:/user/hive/warehouse/jj.db/ssuser/000000_0.avro:0+598480InputFormatClass: org.apache.hadoop.mapred.TextInputFormat
2013-09-29 16:41:13,965 WARN org.apache.hadoop.hive.conf.HiveConf: hive-site.xml not found on CLASSPATH
2013-09-29 16:41:14,166 WARN org.apache.hadoop.io.compress.snappy.LoadSnappy: Snappy native library is available
2013-09-29 16:41:14,166 INFO org.apache.hadoop.io.compress.snappy.LoadSnappy: Snappy native library loaded
2013-09-29 16:41:14,175 INFO org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader: Processing file hdfs://172.16.0.237:9000/user/hive/warehouse/jj.db/ssuser/000000_0.avro
2013-09-29 16:41:14,175 INFO org.apache.hadoop.mapred.MapTask: numReduceTasks: 1
2013-09-29 16:41:14,185 INFO org.apache.hadoop.mapred.MapTask: io.sort.mb = 100
2013-09-29 16:41:14,227 INFO org.apache.hadoop.mapred.MapTask: data buffer = 79691776/99614720
2013-09-29 16:41:14,227 INFO org.apache.hadoop.mapred.MapTask: record buffer = 262144/327680
2013-09-29 16:41:14,236 INFO ExecMapper: maximum memory = 186646528
2013-09-29 16:41:14,236 INFO ExecMapper: conf classpath = [file:/home/qos/app/hadoop-1.2.1/conf/, file:/usr/java/jdk1.7.0_40/lib/tools.jar, file:/home/qos/app/hadoop-1.2.1/, file:/home/qos/app/hadoop-1.2.1/hadoop-core-1.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/asm-3.2.jar, file:/home/qos/app/hadoop-1.2.1/lib/aspectjrt-1.6.11.jar, file:/home/qos/app/hadoop-1.2.1/lib/aspectjtools-1.6.11.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-beanutils-1.7.0.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-beanutils-core-1.8.0.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-cli-1.2.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-codec-1.4.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-collections-3.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-configuration-1.6.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-daemon-1.0.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-digester-1.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-el-1.0.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-httpclient-3.0.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-io-2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-lang-2.4.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-logging-1.1.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-logging-api-1.0.4.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-math-2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-net-3.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/core-3.1.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/derby-10.4.2.0.jar, file:/home/qos/app/hadoop-1.2.1/lib/hadoop-capacity-scheduler-1.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/hadoop-fairscheduler-1.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/hadoop-thriftfs-1.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/hsqldb-1.8.0.10.jar, file:/home/qos/app/hadoop-1.2.1/lib/jackson-core-asl-1.8.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jackson-mapper-asl-1.8.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jasper-compiler-5.5.12.jar, file:/home/qos/app/hadoop-1.2.1/lib/jasper-runtime-5.5.12.jar, file:/home/qos/app/hadoop-1.2.1/lib/jdeb-0.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jersey-core-1.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jersey-json-1.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jersey-server-1.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jets3t-0.6.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/jetty-6.1.26.jar, file:/home/qos/app/hadoop-1.2.1/lib/jetty-util-6.1.26.jar, file:/home/qos/app/hadoop-1.2.1/lib/jsch-0.1.42.jar, file:/home/qos/app/hadoop-1.2.1/lib/junit-4.5.jar, file:/home/qos/app/hadoop-1.2.1/lib/kfs-0.2.2.jar, file:/home/qos/app/hadoop-1.2.1/lib/log4j-1.2.15.jar, file:/home/qos/app/hadoop-1.2.1/lib/mockito-all-1.8.5.jar, file:/home/qos/app/hadoop-1.2.1/lib/oro-2.0.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/servlet-api-2.5-20081211.jar, file:/home/qos/app/hadoop-1.2.1/lib/slf4j-api-1.4.3.jar, file:/home/qos/app/hadoop-1.2.1/lib/slf4j-log4j12-1.4.3.jar, file:/home/qos/app/hadoop-1.2.1/lib/snappy-java-1.1.0-M4.jar, file:/home/qos/app/hadoop-1.2.1/lib/xmlenc-0.52.jar, file:/home/qos/app/hadoop-1.2.1/lib/jsp-2.1/jsp-2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/jsp-2.1/jsp-api-2.1.jar, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/classes, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/distcache/-6906649896783115865_-1711195192_1759672612/172.16.0.237/home/qos/app/hadoop/tmp/mapred/staging/qos/.staging/job_201309271638_0025/libjars/hive-builtins-0.10.0.jar/, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/distcache/2983734166189404195_41183263_1759672650/172.16.0.237/home/qos/app/hadoop/tmp/mapred/staging/qos/.staging/job_201309271638_0025/libjars/hive-function-1.0-SNAPSHOT.jar/, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/]
2013-09-29 16:41:14,236 INFO ExecMapper: thread classpath = [file:/home/qos/app/hadoop-1.2.1/conf/, file:/usr/java/jdk1.7.0_40/lib/tools.jar, file:/home/qos/app/hadoop-1.2.1/, file:/home/qos/app/hadoop-1.2.1/hadoop-core-1.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/asm-3.2.jar, file:/home/qos/app/hadoop-1.2.1/lib/aspectjrt-1.6.11.jar, file:/home/qos/app/hadoop-1.2.1/lib/aspectjtools-1.6.11.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-beanutils-1.7.0.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-beanutils-core-1.8.0.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-cli-1.2.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-codec-1.4.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-collections-3.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-configuration-1.6.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-daemon-1.0.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-digester-1.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-el-1.0.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-httpclient-3.0.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-io-2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-lang-2.4.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-logging-1.1.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-logging-api-1.0.4.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-math-2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/commons-net-3.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/core-3.1.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/derby-10.4.2.0.jar, file:/home/qos/app/hadoop-1.2.1/lib/hadoop-capacity-scheduler-1.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/hadoop-fairscheduler-1.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/hadoop-thriftfs-1.2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/hsqldb-1.8.0.10.jar, file:/home/qos/app/hadoop-1.2.1/lib/jackson-core-asl-1.8.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jackson-mapper-asl-1.8.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jasper-compiler-5.5.12.jar, file:/home/qos/app/hadoop-1.2.1/lib/jasper-runtime-5.5.12.jar, file:/home/qos/app/hadoop-1.2.1/lib/jdeb-0.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jersey-core-1.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jersey-json-1.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jersey-server-1.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/jets3t-0.6.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/jetty-6.1.26.jar, file:/home/qos/app/hadoop-1.2.1/lib/jetty-util-6.1.26.jar, file:/home/qos/app/hadoop-1.2.1/lib/jsch-0.1.42.jar, file:/home/qos/app/hadoop-1.2.1/lib/junit-4.5.jar, file:/home/qos/app/hadoop-1.2.1/lib/kfs-0.2.2.jar, file:/home/qos/app/hadoop-1.2.1/lib/log4j-1.2.15.jar, file:/home/qos/app/hadoop-1.2.1/lib/mockito-all-1.8.5.jar, file:/home/qos/app/hadoop-1.2.1/lib/oro-2.0.8.jar, file:/home/qos/app/hadoop-1.2.1/lib/servlet-api-2.5-20081211.jar, file:/home/qos/app/hadoop-1.2.1/lib/slf4j-api-1.4.3.jar, file:/home/qos/app/hadoop-1.2.1/lib/slf4j-log4j12-1.4.3.jar, file:/home/qos/app/hadoop-1.2.1/lib/snappy-java-1.1.0-M4.jar, file:/home/qos/app/hadoop-1.2.1/lib/xmlenc-0.52.jar, file:/home/qos/app/hadoop-1.2.1/lib/jsp-2.1/jsp-2.1.jar, file:/home/qos/app/hadoop-1.2.1/lib/jsp-2.1/jsp-api-2.1.jar, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/classes, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/jars/, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/distcache/-6906649896783115865_-1711195192_1759672612/172.16.0.237/home/qos/app/hadoop/tmp/mapred/staging/qos/.staging/job_201309271638_0025/libjars/hive-builtins-0.10.0.jar/, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/distcache/2983734166189404195_41183263_1759672650/172.16.0.237/home/qos/app/hadoop/tmp/mapred/staging/qos/.staging/job_201309271638_0025/libjars/hive-function-1.0-SNAPSHOT.jar/, file:/home/qos/app/hadoop-1.2.1/tmp/mapred/local/taskTracker/qos/jobcache/job_201309271638_0025/attempt_201309271638_0025_m_000000_0/work/]
2013-09-29 16:41:14,280 INFO org.apache.hadoop.hive.ql.exec.MapOperator: Adding alias ssuser to work list for file hdfs://172.16.0.237:9000/user/hive/warehouse/jj.db/ssuser
2013-09-29 16:41:14,283 INFO org.apache.hadoop.hive.ql.exec.MapOperator: dump TS struct<siteid:string,day:string,time:string,ip:string,clientid:string,viewid:string,msgid:int,eventtype:string,playtime:int,devicetype:string,apptype:string,updateinterval:int,convention:string,appversion:string,streamurl:string,streamdescription:string,streamtype:int,streamlength:int,os:string,player:string,productid:string,networktype:string,bandwidth:int,bitrate:int,switchmethod:int,bytesloaded:map<string,bigint>,bytesloadeddelta:map<string,bigint>,dropframecount:int,windowmode:string,bufferlength:int,buffertime:int,isairplay:int,airplayduration:int,userid:string,progid:int,progtype:string,gameid:int,gamedate:string,hometeam:string,awayteam:string,code:int,dt:string>
2013-09-29 16:41:14,284 INFO ExecMapper:
<MAP>Id =7
<Children>
<TS>Id =3
<Children>
<SEL>Id =2
<Children>
<GBY>Id =1
<Children>
<RS>Id =0
<Parent>Id = 1 null<\Parent>
<\RS>
<\Children>
<Parent>Id = 2 null<\Parent>
<\GBY>
<\Children>
<Parent>Id = 3 null<\Parent>
<\SEL>
<\Children>
<Parent>Id = 7 null<\Parent>
<\TS>
<\Children>
<\MAP>
2013-09-29 16:41:14,284 INFO org.apache.hadoop.hive.ql.exec.MapOperator: Initializing Self 7 MAP
2013-09-29 16:41:14,284 INFO org.apache.hadoop.hive.ql.exec.TableScanOperator: Initializing Self 3 TS
2013-09-29 16:41:14,284 INFO org.apache.hadoop.hive.ql.exec.TableScanOperator: Operator 3 TS initialized
2013-09-29 16:41:14,284 INFO org.apache.hadoop.hive.ql.exec.TableScanOperator: Initializing children of 3 TS
2013-09-29 16:41:14,284 INFO org.apache.hadoop.hive.ql.exec.SelectOperator: Initializing child 2 SEL
2013-09-29 16:41:14,284 INFO org.apache.hadoop.hive.ql.exec.SelectOperator: Initializing Self 2 SEL
2013-09-29 16:41:14,287 INFO org.apache.hadoop.hive.ql.exec.SelectOperator: SELECT struct<siteid:string,day:string,time:string,ip:string,clientid:string,viewid:string,msgid:int,eventtype:string,playtime:int,devicetype:string,apptype:string,updateinterval:int,convention:string,appversion:string,streamurl:string,streamdescription:string,streamtype:int,streamlength:int,os:string,player:string,productid:string,networktype:string,bandwidth:int,bitrate:int,switchmethod:int,bytesloaded:map<string,bigint>,bytesloadeddelta:map<string,bigint>,dropframecount:int,windowmode:string,bufferlength:int,buffertime:int,isairplay:int,airplayduration:int,userid:string,progid:int,progtype:string,gameid:int,gamedate:string,hometeam:string,awayteam:string,code:int,dt:string>
2013-09-29 16:41:14,289 INFO org.apache.hadoop.hive.ql.exec.SelectOperator: Operator 2 SEL initialized
2013-09-29 16:41:14,289 INFO org.apache.hadoop.hive.ql.exec.SelectOperator: Initializing children of 2 SEL
2013-09-29 16:41:14,289 INFO org.apache.hadoop.hive.ql.exec.GroupByOperator: Initializing child 1 GBY
2013-09-29 16:41:14,289 INFO org.apache.hadoop.hive.ql.exec.GroupByOperator: Initializing Self 1 GBY
2013-09-29 16:41:14,313 INFO org.apache.hadoop.mapred.TaskLogsTruncater: Initializing logs' truncater with mapRetainSize=-1 and reduceRetainSize=-1
2013-09-29 16:41:14,332 INFO org.apache.hadoop.io.nativeio.NativeIO: Initialized cache for UID to User mapping with a cache timeout of 14400 seconds.
2013-09-29 16:41:14,333 INFO org.apache.hadoop.io.nativeio.NativeIO: Got UserName qos for UID 500 from the native implementation
2013-09-29 16:41:14,335 WARN org.apache.hadoop.mapred.Child: Error running child
java.lang.RuntimeException: Error in configuring object
at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93)
at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64)
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:426)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:366)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1190)
at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
... 9 more
Caused by: java.lang.RuntimeException: Error in configuring object
at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93)
at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64)
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117)
at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34)
... 14 more
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
... 17 more
Caused by: java.lang.RuntimeException: Map operator initialization failed
at org.apache.hadoop.hive.ql.exec.ExecMapper.configure(ExecMapper.java:121)
... 22 more
Caused by: java.lang.NullPointerException
at org.apache.hadoop.hive.ql.exec.GroupByOperator.initializeOp(GroupByOperator.java:368)
at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:360)
at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:436)
at org.apache.hadoop.hive.ql.exec.Operator.initializeChildren(Operator.java:392)
at org.apache.hadoop.hive.ql.exec.SelectOperator.initializeOp(SelectOperator.java:62)
at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:360)
at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:436)
at org.apache.hadoop.hive.ql.exec.Operator.initializeChildren(Operator.java:392)
at org.apache.hadoop.hive.ql.exec.TableScanOperator.initializeOp(TableScanOperator.java:166)
at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:360)
at org.apache.hadoop.hive.ql.exec.MapOperator.initializeOp(MapOperator.java:441)
at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:360)
at org.apache.hadoop.hive.ql.exec.ExecMapper.configure(ExecMapper.java:98)
... 22 more
2013-09-29 16:41:14,338 INFO org.apache.hadoop.mapred.Task: Runnning cleanup for the task
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment