shrijeet

diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
index 2cc1b04..c08a55e 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
@@ -209,6 +209,7 @@ public class HBaseClient {
    * socket connected to a remote address.  Calls are multiplexed through this
    * socket: responses may be delivered out of order. */
   private class Connection extends Thread {
+    protected static final long DEFAULT_CLEAN_INTERVAL = -1; // disabled by default
     private ConnectionId remoteId;

shrijeet

/**
 * Copyright 2010 The Apache Software Foundation
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at

shrijeet

select pid as pid, 
       sum(if(bid is not null and bid <> '', 1, 0)) as bids, 
       sum(1) as requests
from table
where data_date = 20120110
and (pid = 15368 or pid = 15369 or pid = 15370)
group by pid, 
       sum(if(bid is not null and bid <> '', 1, 0)), 
       sum(1)

shrijeet

FAILED: Hive Internal Error: java.lang.NullPointerException(null)
java.lang.NullPointerException
	at org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc.newInstance(ExprNodeGenericFuncDesc.java:214)
	at org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory$DefaultExprProcessor.getXpathOrFuncExprNodeDesc(TypeCheckProcFactory.java:684)
	at org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory$DefaultExprProcessor.process(TypeCheckProcFactory.java:805)
	at org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher.dispatch(DefaultRuleDispatcher.java:89)
	at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatch(DefaultGraphWalker.java:88)
	at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.walk(DefaultGraphWalker.java:125)
	at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.startWalking(DefaultGraphWalker.java:102)
	at org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory.genExprNode(TypeCheckProcFactory.java:161)

shrijeet

diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java
index a3e40f7..7674af4 100644                                                   
--- ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java                  
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java                  
@@ -381,7 +381,7 @@ public class MapRedTask extends ExecDriver implements Serializable {
           .printInfo("Number of reduce tasks is set to 0 since there's no reduce operator");
       work.setNumReduceTasks(Integer.valueOf(0));                              
     } else {                                                                   
-      if (numReducersFromWork >= 0) {                                          
+      if (numReducersFromWork > 0) {                                           

shrijeet

Hive Version: Hive 0.8 (last commit SHA  b581a6192b8d4c544092679d05f45b2e50d42b45 ) 

Hadoop version : chd3u0

I am trying to use the hive merge small file feature by setting all the necessary params.
I am disabling use of CombineHiveInputFormat since my input is compressed text. 

hive> set mapred.min.split.size.per.node=1000000000;
hive> set mapred.min.split.size.per.rack=1000000000;
hive> set mapred.max.split.size=1000000000;

shrijeet

stdout logs
0.292: [GC [PSYoungGen: 3072K->512K(3584K)] 3072K->552K(8896K), 0.0028970 secs] [Times: user=0.01 sys=0.01, real=0.00 secs] 
0.520: [GC [PSYoungGen: 3583K->480K(3584K)] 3623K->880K(8896K), 0.0018630 secs] [Times: user=0.01 sys=0.00, real=0.01 secs] 
0.670: [GC [PSYoungGen: 3552K->512K(3584K)] 3952K->1328K(8896K), 0.0019800 secs] [Times: user=0.00 sys=0.00, real=0.01 secs] 
0.825: [GC [PSYoungGen: 3584K->512K(6656K)] 4400K->1550K(11968K), 0.0045470 secs] [Times: user=0.01 sys=0.01, real=0.00 secs] 
1.164: [GC [PSYoungGen: 6656K->512K(6656K)] 7694K->2742K(11968K), 0.0035330 secs] [Times: user=0.01 sys=0.00, real=0.00 secs] 
1.390: [GC [PSYoungGen: 6656K->528K(13888K)] 8886K->2758K(19200K), 0.0053980 secs] [Times: user=0.01 sys=0.00, real=0.01 secs] 
1.802: [GC [PSYoungGen: 12816K->1226K(13888K)] 15046K->3737K(19200K), 0.0053020 secs] [Times: user=0.01 sys=0.00, real=0.00 secs] 
2.150: [GC [PSYoungGen: 13514K->1258K(26304K)] 16025K->3817K(31616K), 0.0036380 secs] [Times: user=0.01 sys=0.00, real=0.01

shrijeet

0.814: [GC [PSYoungGen: 3072K->480K(3584K)] 3072K->504K(8896K), 0.0024990 secs] [Times: user=0.00 sys=0.00, real=0.01 secs] 
1.194: [GC [PSYoungGen: 3549K->512K(3584K)] 3573K->784K(8896K), 0.0025040 secs] [Times: user=0.01 sys=0.00, real=0.01 secs] 
1.474: [GC [PSYoungGen: 3584K->480K(3584K)] 3856K->1216K(8896K), 0.0035110 secs] [Times: user=0.01 sys=0.00, real=0.01 secs] 
1.680: [GC [PSYoungGen: 3552K->512K(3584K)] 4288K->1376K(8896K), 0.0044320 secs] [Times: user=0.01 sys=0.00, real=0.00 secs] 
2.023: [GC [PSYoungGen: 3584K->512K(3584K)] 4448K->1758K(8896K), 0.0059910 secs] [Times: user=0.01 sys=0.00, real=0.01 secs] 
2.308: [GC [PSYoungGen: 3584K->1056K(6912K)] 4830K->2558K(12224K), 0.0035610 secs] [Times: user=0.00 sys=0.00, real=0.00 secs] 
2.717: [GC [PSYoungGen: 6880K->640K(7232K)] 8382K->2367K(12544K), 0.0045720 secs] [Times: user=0.01 sys=0.00, real=0.00 secs] 
3.164: [GC [PSYoungGen: 6464K->1226K(12864K)] 8191K->3017K(18176K), 0.0153380 secs] [Times: user=0.01 sys=0.02, real=0.02 secs] 
3.936: [GC [

shrijeet

0.267: [GC [PSYoungGen: 3072K->480K(3584K)] 3072K->488K(8896K), 0.0086210 secs] [Times: user=0.02 sys=0.00, real=0.01 secs] 
0.499: [GC [PSYoungGen: 3552K->480K(6656K)] 3560K->772K(11968K), 0.0122940 secs] [Times: user=0.01 sys=0.00, real=0.01 secs] 
0.825: [GC [PSYoungGen: 6624K->512K(6656K)] 6916K->1084K(11968K), 0.0082890 secs] [Times: user=0.01 sys=0.00, real=0.01 secs] 
1.060: [GC [PSYoungGen: 6656K->512K(12800K)] 7228K->2184K(18112K), 0.0035620 secs] [Times: user=0.01 sys=0.00, real=0.00 secs] 
1.447: [GC [PSYoungGen: 12800K->496K(12800K)] 14472K->3157K(18112K), 0.0115740 secs] [Times: user=0.01 sys=0.02, real=0.02 secs] 
1.835: [GC [PSYoungGen: 12784K->1008K(26240K)] 15445K->3669K(31552K), 0.0845520 secs] [Times: user=0.01 sys=0.00, real=0.09 secs] 
2.359: [GC [PSYoungGen: 25584K->1280K(26368K)] 28245K->4118K(31680K), 0.0428030 secs] [Times: user=0.06 sys=0.00, real=0.04 secs] 
2.651: [GC [PSYoungGen: 25856K->1472K(50816K)] 28694K->4398K(56128K), 0.0038860 secs] [Times: user=0.01 sys=0.00, real=0.00 se

shrijeet

0.212: [GC [PSYoungGen: 3072K->480K(3584K)] 3072K->496K(8896K), 0.0290900 secs] [Times: user=0.00 sys=0.00, real=0.03 secs] 
0.416: [GC [PSYoungGen: 3552K->496K(6656K)] 3568K->748K(11968K), 0.0100310 secs] [Times: user=0.00 sys=0.00, real=0.01 secs] 
0.671: [GC [PSYoungGen: 6640K->496K(6656K)] 6892K->1084K(11968K), 0.0069680 secs] [Times: user=0.01 sys=0.00, real=0.00 secs] 
0.894: [GC [PSYoungGen: 6640K->496K(12800K)] 7228K->2228K(18112K), 0.0104630 secs] [Times: user=0.02 sys=0.01, real=0.01 secs] 
1.547: [GC [PSYoungGen: 12784K->512K(12800K)] 14516K->3243K(18112K), 0.0035770 secs] [Times: user=0.01 sys=0.00, real=0.00 secs] 
1.834: [GC [PSYoungGen: 12800K->1120K(26368K)] 15531K->3851K(31680K), 0.0064140 secs] [Times: user=0.01 sys=0.00, real=0.00 secs] 
2.253: [GC [PSYoungGen: 25696K->1312K(26432K)] 28427K->4179K(31744K), 0.0050690 secs] [Times: user=0.02 sys=0.01, real=0.00 secs] 
2.631: [GC [PSYoungGen: 25888K->1504K(50880K)] 28755K->4476K(56192K), 0.0076770 secs] [Times: user=0.01 sys=0.01, real=0.00 se