Created
April 14, 2015 19:24
-
-
Save mxk1235/6bec63717bb09b40f82f to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 19:19:31.996 ERROR [RegionClient.exceptionCaught] - Unexpected exception from downstream on [id: 0xa3031187, /10.132.7.1:39229 => /10.132.2.12:60020] | |
| java.io.IOException: Connection reset by peer | |
| at sun.nio.ch.FileDispatcherImpl.read0(Native Method) ~[na:1.8.0_20] | |
| at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) ~[na:1.8.0_20] | |
| at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[na:1.8.0_20] | |
| at sun.nio.ch.IOUtil.read(IOUtil.java:192) ~[na:1.8.0_20] | |
| at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379) ~[na:1.8.0_20] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:64) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) [netty-3.9.4.Final.jar:na] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_20] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_20] | |
| at java.lang.Thread.run(Thread.java:745) [na:1.8.0_20] | |
| 19:19:31.997 INFO [HBaseClient.removeClientFromCache] - Lost connection with the .META. region | |
| 19:19:32.007 INFO [HBaseClient.handleNSRE] - There are now 500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.015 INFO [HBaseClient.handleNSRE] - There are now 1000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.029 INFO [HBaseClient.handleNSRE] - There are now 1500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.034 INFO [HBaseClient.handleNSRE] - There are now 2000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.037 INFO [HBaseClient.handleNSRE] - There are now 2500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.040 INFO [HBaseClient.handleNSRE] - There are now 3000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.043 INFO [HBaseClient.handleNSRE] - There are now 3500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.046 INFO [HBaseClient.handleNSRE] - There are now 4000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.049 INFO [HBaseClient.handleNSRE] - There are now 4500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.053 INFO [HBaseClient.handleNSRE] - There are now 5000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.056 INFO [HBaseClient.handleNSRE] - There are now 5500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.059 INFO [HBaseClient.handleNSRE] - There are now 6000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.063 INFO [HBaseClient.handleNSRE] - There are now 6500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.066 INFO [HBaseClient.handleNSRE] - There are now 7000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.070 INFO [HBaseClient.handleNSRE] - There are now 7500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.074 INFO [HBaseClient.handleNSRE] - There are now 8000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.077 INFO [HBaseClient.handleNSRE] - There are now 8500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.081 INFO [HBaseClient.handleNSRE] - There are now 9000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.085 WARN [HBaseClient.handleNSRE] - There are now 9500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.421 INFO [ZooKeeper.<init>] - Initiating client connection, connectString=hadoop-nn01.prod.sjc.cyanogen.net,hadoop-nn02.prod.sjc.cyanogen.net,hadoop-zk01.prod.sjc.cyanogen.net sessionTimeout=5000 watcher=org.hbase.async.HBaseClient$ZKClient@6500df86 | |
| 19:19:32.429 INFO [ClientCnxn.startConnect] - Opening socket connection to server hadoop-nn01.prod.sjc.cyanogen.net/10.132.2.154:2181 | |
| 19:19:32.429 INFO [HBaseClient.getDeferredRoot] - Need to find the .META. region | |
| 19:19:32.435 INFO [ClientCnxn.primeConnection] - Socket connection established to hadoop-nn01.prod.sjc.cyanogen.net/10.132.2.154:2181, initiating session | |
| 19:19:32.440 INFO [ClientCnxn.readConnectResult] - Session establishment complete on server hadoop-nn01.prod.sjc.cyanogen.net/10.132.2.154:2181, sessionid = 0x14aabe20e043e55, negotiated timeout = 6000 | |
| 19:19:32.443 INFO [HBaseClient.handleMetaZnode] - Connecting to .META. region @ 10.132.2.12:60020 | |
| 19:19:32.451 INFO [ZooKeeper.close] - Session: 0x14aabe20e043e55 closed | |
| 19:19:32.452 INFO [ClientCnxn.run] - EventThread shut down | |
| 19:19:32.451 INFO [HBaseClient.discoverRegion] - Added client for region RegionInfo(table="tsdb-uid", region_name="tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112.", stop_key=""), which was added to the regions cache. Now we know that RegionClient@1981764605(chan=[id: 0x49cac5ea, /10.132.7.1:39257 => /10.132.2.12:60020], #pending_rpcs=0, #batched=0, #rpcs_inflight=0) is hosting 1 region. | |
| 19:19:32.456 INFO [HBaseClient.handleNSRE] - There are now 500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.457 INFO [HBaseClient.invalidateRegionCache] - Invalidated cache for RegionInfo(table="tsdb-uid", region_name="tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112.", stop_key="") as RegionClient@1981764605(chan=[id: 0x49cac5ea, /10.132.7.1:39257 => /10.132.2.12:60020], #pending_rpcs=0, #batched=0, #rpcs_inflight=1) seems to be splitting or closing it. | |
| 19:19:32.459 INFO [HBaseClient.handleNSRE] - There are now 1000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.461 INFO [HBaseClient.handleNSRE] - There are now 1500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.463 INFO [HBaseClient.handleNSRE] - There are now 2000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.464 INFO [HBaseClient.handleNSRE] - There are now 2500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.466 INFO [HBaseClient.handleNSRE] - There are now 3000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.468 INFO [HBaseClient.handleNSRE] - There are now 3500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.471 INFO [HBaseClient.handleNSRE] - There are now 4000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.475 INFO [HBaseClient.handleNSRE] - There are now 4500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.477 INFO [HBaseClient.handleNSRE] - There are now 5000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.480 INFO [HBaseClient.handleNSRE] - There are now 5500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.482 INFO [HBaseClient.handleNSRE] - There are now 6000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.486 INFO [HBaseClient.handleNSRE] - There are now 6500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.495 INFO [HBaseClient.handleNSRE] - There are now 7500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.491 INFO [HBaseClient.handleNSRE] - There are now 7000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.500 INFO [HBaseClient.handleNSRE] - There are now 8000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.504 INFO [HBaseClient.handleNSRE] - There are now 8500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.509 INFO [HBaseClient.handleNSRE] - There are now 9000 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:32.514 WARN [HBaseClient.handleNSRE] - There are now 9500 RPCs pending due to NSRE on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." | |
| 19:19:35.750 ERROR [RpcHandler.logError] - [id: 0x5394e94f, /172.16.88.4:56163 => /10.132.7.1:4242] Unexpected exception caught while serving [stats] | |
| java.lang.RuntimeException: Shouldn't be here | |
| at net.opentsdb.core.TSDB.collectStats(TSDB.java:411) ~[tsdb-2.2.0-SNAPSHOT.jar:] | |
| at net.opentsdb.tsd.StatsRpc.doCollectStats(StatsRpc.java:106) ~[tsdb-2.2.0-SNAPSHOT.jar:] | |
| at net.opentsdb.tsd.StatsRpc.execute(StatsRpc.java:51) ~[tsdb-2.2.0-SNAPSHOT.jar:] | |
| at net.opentsdb.tsd.RpcHandler.handleTelnetRpc(RpcHandler.java:208) ~[tsdb-2.2.0-SNAPSHOT.jar:] | |
| at net.opentsdb.tsd.RpcHandler.messageReceived(RpcHandler.java:178) ~[tsdb-2.2.0-SNAPSHOT.jar:] | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:70) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.SimpleChannelHandler.messageReceived(SimpleChannelHandler.java:142) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.SimpleChannelHandler.handleUpstream(SimpleChannelHandler.java:88) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler.handleUpstream(IdleStateAwareChannelHandler.java:36) [netty-3.9.4.Final.jar:na] | |
| at net.opentsdb.tsd.ConnectionManager.handleUpstream(ConnectionManager.java:90) [tsdb-2.2.0-SNAPSHOT.jar:] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.handler.timeout.IdleStateHandler.messageReceived(IdleStateHandler.java:294) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) [netty-3.9.4.Final.jar:na] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_20] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_20] | |
| at java.lang.Thread.run(Thread.java:745) [na:1.8.0_20] | |
| Caused by: org.hbase.async.PleaseThrottleException: 10000 RPCs waiting on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." to come back online | |
| at org.hbase.async.HBaseClient.handleNSRE(HBaseClient.java:2410) ~[asynchbase-1.6.0.jar:na] | |
| at org.hbase.async.RegionClient.failOrRetryRpcs(RegionClient.java:1071) ~[asynchbase-1.6.0.jar:na] | |
| at org.hbase.async.RegionClient.cleanup(RegionClient.java:1033) ~[asynchbase-1.6.0.jar:na] | |
| at org.hbase.async.RegionClient.channelDisconnected(RegionClient.java:1010) ~[asynchbase-1.6.0.jar:na] | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) [netty-3.9.4.Final.jar:na] | |
| at org.hbase.async.RegionClient.handleUpstream(RegionClient.java:1082) ~[asynchbase-1.6.0.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) [netty-3.9.4.Final.jar:na] | |
| at org.hbase.async.HBaseClient$RegionClientPipeline.sendUpstream(HBaseClient.java:2677) ~[asynchbase-1.6.0.jar:na] | |
| at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.close(AbstractNioWorker.java:360) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioClientSocketPipelineSink.eventSunk(NioClientSocketPipelineSink.java:58) ~[netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:574) [netty-3.9.4.Final.jar:na] | |
| at org.hbase.async.HBaseClient$RegionClientPipeline.sendDownstream(HBaseClient.java:2667) ~[asynchbase-1.6.0.jar:na] | |
| at org.jboss.netty.channel.Channels.close(Channels.java:812) [netty-3.9.4.Final.jar:na] | |
| at org.hbase.async.RegionClient.exceptionCaught(RegionClient.java:1098) ~[asynchbase-1.6.0.jar:na] | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:112) [netty-3.9.4.Final.jar:na] | |
| at org.hbase.async.RegionClient.handleUpstream(RegionClient.java:1082) ~[asynchbase-1.6.0.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) [netty-3.9.4.Final.jar:na] | |
| at org.hbase.async.HBaseClient$RegionClientPipeline.sendUpstream(HBaseClient.java:2677) ~[asynchbase-1.6.0.jar:na] | |
| at org.jboss.netty.channel.Channels.fireExceptionCaught(Channels.java:525) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:74) [netty-3.9.4.Final.jar:na] | |
| ... 9 common frames omitted | |
| Caused by: org.hbase.async.NotServingRegionException: Connection reset: [id: 0xa3031187, /10.132.7.1:39229 :> /10.132.2.12:60020] got disconnected | |
| Caused by RPC: GetRequest(table="tsdb-uid", key="\x00", family="id", qualifiers=["metrics", "tagk", "tagv"], attempt=0, region=RegionInfo(table="tsdb-uid", region_name="tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112.", stop_key="")) | |
| at org.hbase.async.RegionClient.failOrRetryRpcs(RegionClient.java:1067) ~[asynchbase-1.6.0.jar:na] | |
| ... 30 common frames omitted | |
| 19:19:35.759 ERROR [RpcHandler.logError] - [id: 0x694a8840, /172.16.88.4:55935 => /10.132.7.1:4242] Unexpected exception caught while serving [stats] | |
| java.lang.RuntimeException: Shouldn't be here | |
| at net.opentsdb.core.TSDB.collectStats(TSDB.java:411) ~[tsdb-2.2.0-SNAPSHOT.jar:] | |
| at net.opentsdb.tsd.StatsRpc.doCollectStats(StatsRpc.java:106) ~[tsdb-2.2.0-SNAPSHOT.jar:] | |
| at net.opentsdb.tsd.StatsRpc.execute(StatsRpc.java:51) ~[tsdb-2.2.0-SNAPSHOT.jar:] | |
| at net.opentsdb.tsd.RpcHandler.handleTelnetRpc(RpcHandler.java:208) ~[tsdb-2.2.0-SNAPSHOT.jar:] | |
| at net.opentsdb.tsd.RpcHandler.messageReceived(RpcHandler.java:178) ~[tsdb-2.2.0-SNAPSHOT.jar:] | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:70) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.SimpleChannelHandler.messageReceived(SimpleChannelHandler.java:142) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.SimpleChannelHandler.handleUpstream(SimpleChannelHandler.java:88) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler.handleUpstream(IdleStateAwareChannelHandler.java:36) [netty-3.9.4.Final.jar:na] | |
| at net.opentsdb.tsd.ConnectionManager.handleUpstream(ConnectionManager.java:90) [tsdb-2.2.0-SNAPSHOT.jar:] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.handler.timeout.IdleStateHandler.messageReceived(IdleStateHandler.java:294) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) [netty-3.9.4.Final.jar:na] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_20] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_20] | |
| at java.lang.Thread.run(Thread.java:745) [na:1.8.0_20] | |
| Caused by: org.hbase.async.PleaseThrottleException: 10000 RPCs waiting on "tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112." to come back online | |
| at org.hbase.async.HBaseClient.handleNSRE(HBaseClient.java:2410) ~[asynchbase-1.6.0.jar:na] | |
| at org.hbase.async.RegionClient.failOrRetryRpcs(RegionClient.java:1071) ~[asynchbase-1.6.0.jar:na] | |
| at org.hbase.async.RegionClient.cleanup(RegionClient.java:1033) ~[asynchbase-1.6.0.jar:na] | |
| at org.hbase.async.RegionClient.channelDisconnected(RegionClient.java:1010) ~[asynchbase-1.6.0.jar:na] | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) [netty-3.9.4.Final.jar:na] | |
| at org.hbase.async.RegionClient.handleUpstream(RegionClient.java:1082) ~[asynchbase-1.6.0.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) [netty-3.9.4.Final.jar:na] | |
| at org.hbase.async.HBaseClient$RegionClientPipeline.sendUpstream(HBaseClient.java:2677) ~[asynchbase-1.6.0.jar:na] | |
| at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.close(AbstractNioWorker.java:360) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioClientSocketPipelineSink.eventSunk(NioClientSocketPipelineSink.java:58) ~[netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:574) [netty-3.9.4.Final.jar:na] | |
| at org.hbase.async.HBaseClient$RegionClientPipeline.sendDownstream(HBaseClient.java:2667) ~[asynchbase-1.6.0.jar:na] | |
| at org.jboss.netty.channel.Channels.close(Channels.java:812) [netty-3.9.4.Final.jar:na] | |
| at org.hbase.async.RegionClient.exceptionCaught(RegionClient.java:1098) ~[asynchbase-1.6.0.jar:na] | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:112) [netty-3.9.4.Final.jar:na] | |
| at org.hbase.async.RegionClient.handleUpstream(RegionClient.java:1082) ~[asynchbase-1.6.0.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) [netty-3.9.4.Final.jar:na] | |
| at org.hbase.async.HBaseClient$RegionClientPipeline.sendUpstream(HBaseClient.java:2677) ~[asynchbase-1.6.0.jar:na] | |
| at org.jboss.netty.channel.Channels.fireExceptionCaught(Channels.java:525) [netty-3.9.4.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:74) [netty-3.9.4.Final.jar:na] | |
| ... 9 common frames omitted | |
| Caused by: org.hbase.async.NotServingRegionException: Connection reset: [id: 0xa3031187, /10.132.7.1:39229 :> /10.132.2.12:60020] got disconnected | |
| Caused by RPC: GetRequest(table="tsdb-uid", key="\x00", family="id", qualifiers=["metrics", "tagk", "tagv"], attempt=0, region=RegionInfo(table="tsdb-uid", region_name="tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112.", stop_key="")) | |
| at org.hbase.async.RegionClient.failOrRetryRpcs(RegionClient.java:1067) ~[asynchbase-1.6.0.jar:na] | |
| ... 30 common frames omitted | |
| 19:19:35.759 INFO [ConnectionManager.handleUpstream] - [id: 0x272bedb2, /127.0.0.1:52197 => /127.0.0.1:4242] BOUND: /127.0.0.1:4242 | |
| 19:19:35.759 INFO [ConnectionManager.handleUpstream] - [id: 0x272bedb2, /127.0.0.1:52197 => /127.0.0.1:4242] CONNECTED: /127.0.0.1:52197 | |
| 19:19:35.760 INFO [ConnectionManager.handleUpstream] - [id: 0x15ec4f7f, /127.0.0.1:52201 => /127.0.0.1:4242] BOUND: /127.0.0.1:4242 | |
| 19:19:35.760 INFO [ConnectionManager.handleUpstream] - [id: 0x15ec4f7f, /127.0.0.1:52201 => /127.0.0.1:4242] CONNECTED: /127.0.0.1:52201 | |
| 19:19:35.760 INFO [ConnectionManager.handleUpstream] - [id: 0x694a8840, /172.16.88.4:55935 :> /10.132.7.1:4242] DISCONNECTED | |
| 19:19:35.760 INFO [ConnectionManager.handleUpstream] - [id: 0x694a8840, /172.16.88.4:55935 :> /10.132.7.1:4242] UNBOUND | |
| 19:19:35.761 INFO [ConnectionManager.handleUpstream] - [id: 0x694a8840, /172.16.88.4:55935 :> /10.132.7.1:4242] CLOSED | |
| 19:19:35.762 INFO [ConnectionManager.handleUpstream] - [id: 0x15ec4f7f, /127.0.0.1:52201 :> /127.0.0.1:4242] DISCONNECTED | |
| 19:19:35.762 INFO [ConnectionManager.handleUpstream] - [id: 0x15ec4f7f, /127.0.0.1:52201 :> /127.0.0.1:4242] UNBOUND | |
| 19:19:35.763 INFO [ConnectionManager.handleUpstream] - [id: 0x15ec4f7f, /127.0.0.1:52201 :> /127.0.0.1:4242] CLOSED | |
| 19:19:35.763 INFO [HttpQuery.logInfo] - [id: 0x15ec4f7f, /127.0.0.1:52201 :> /127.0.0.1:4242] HTTP /api/version done in 1ms | |
| 19:19:35.765 INFO [ConnectionManager.handleUpstream] - [id: 0x272bedb2, /127.0.0.1:52197 :> /127.0.0.1:4242] DISCONNECTED | |
| 19:19:35.765 INFO [ConnectionManager.handleUpstream] - [id: 0x272bedb2, /127.0.0.1:52197 :> /127.0.0.1:4242] UNBOUND | |
| 19:19:35.769 INFO [ConnectionManager.handleUpstream] - [id: 0x272bedb2, /127.0.0.1:52197 :> /127.0.0.1:4242] CLOSED | |
| 19:19:35.770 INFO [HttpQuery.logInfo] - [id: 0x272bedb2, /127.0.0.1:52197 :> /127.0.0.1:4242] HTTP /api/version done in 6ms | |
| 19:19:38.645 INFO [HBaseClient.discoverRegion] - Added client for region RegionInfo(table="tsdb-uid", region_name="tsdb-uid,,1414486416834.fe60c65edcbbef1dfb1c21daf0c8d112.", stop_key=""), which was added to the regions cache. Now we know that RegionClient@1981764605(chan=[id: 0x49cac5ea, /10.132.7.1:39257 => /10.132.2.12:60020], #pending_rpcs=0, #batched=0, #rpcs_inflight=938) is hosting 1 region. | |
| 19:20:01.822 INFO [ConnectionManager.handleUpstream] - [id: 0xd73f346d, /127.0.0.1:52206 => /127.0.0.1:4242] OPEN | |
| 19:20:01.822 INFO [ConnectionManager.handleUpstream] - [id: 0xd73f346d, /127.0.0.1:52206 => /127.0.0.1:4242] BOUND: /127.0.0.1:4242 | |
| 19:20:01.823 INFO [ConnectionManager.handleUpstream] - [id: 0xd73f346d, /127.0.0.1:52206 => /127.0.0.1:4242] CONNECTED: /127.0.0.1:52206 | |
| 19:20:01.825 INFO [ConnectionManager.handleUpstream] - [id: 0xd73f346d, /127.0.0.1:52206 :> /127.0.0.1:4242] DISCONNECTED | |
| 19:20:01.825 INFO [ConnectionManager.handleUpstream] - [id: 0xd73f346d, /127.0.0.1:52206 :> /127.0.0.1:4242] UNBOUND | |
| 19:20:01.825 INFO [ConnectionManager.handleUpstream] - [id: 0xd73f346d, /127.0.0.1:52206 :> /127.0.0.1:4242] CLOSED | |
| 19:20:01.826 INFO [HttpQuery.logInfo] - [id: 0xd73f346d, /127.0.0.1:52206 :> /127.0.0.1:4242] HTTP /api/version done in 1ms | |
| 19:21:01.821 INFO [ConnectionManager.handleUpstream] - [id: 0xb7e9fede, /127.0.0.1:52207 => /127.0.0.1:4242] OPEN | |
| 19:22:01.822 INFO [ConnectionManager.handleUpstream] - [id: 0xa1b8a8c4, /127.0.0.1:52208 => /127.0.0.1:4242] OPEN | |
| 19:22:01.823 INFO [ConnectionManager.handleUpstream] - [id: 0xa1b8a8c4, /127.0.0.1:52208 => /127.0.0.1:4242] BOUND: /127.0.0.1:4242 | |
| 19:22:01.824 INFO [ConnectionManager.handleUpstream] - [id: 0xa1b8a8c4, /127.0.0.1:52208 => /127.0.0.1:4242] CONNECTED: /127.0.0.1:52208 | |
| 19:22:01.826 INFO [ConnectionManager.handleUpstream] - [id: 0xa1b8a8c4, /127.0.0.1:52208 :> /127.0.0.1:4242] DISCONNECTED | |
| 19:22:01.826 INFO [ConnectionManager.handleUpstream] - [id: 0xa1b8a8c4, /127.0.0.1:52208 :> /127.0.0.1:4242] UNBOUND | |
| 19:22:01.827 INFO [ConnectionManager.handleUpstream] - [id: 0xa1b8a8c4, /127.0.0.1:52208 :> /127.0.0.1:4242] CLOSED | |
| 19:22:01.827 INFO [HttpQuery.logInfo] - [id: 0xa1b8a8c4, /127.0.0.1:52208 :> /127.0.0.1:4242] HTTP /api/version done in 2ms |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment