Skip to content

Instantly share code, notes, and snippets.

@haridsv haridsv/1.error.txt
Last active Aug 29, 2015

Embed
What would you like to do?
OpenTSDB errors under load
2015-04-27 13:27:17,030 ERROR [New I/O worker #67] RegionClient: Uncaught error during de-serialization of MultiAction(batch=[PutRequest(table="tsdb", key=[0, 0, 22, 85, 27, -35, -64, 0, 0, 1, 0, 0, -64], family="t", qualifiers=[[17, -121]], values=[[0, 0, 0, 10, -64, 92, -47, 108]], timestamp=9223372036854775807, lockid=-1, durable=true, bufferable=true, attempt=2, region=RegionInfo(table="tsdb", region_name="tsdb,,1430141025970.93e30e6cb050703b91b3aae198c79e3c.", stop_key=[0, 0, 78, 85, 27, -105, 112, 0, 0, 1, 0, 0, -33, 0, 0, 13, 0, 0, 111, 0, 0, 14, 0, 0, 121, 0, 0, 15, 0, 0, 103, 0, 0, 16, 0, 0, 87])), PutRequest(table="tsdb", key=[0, 0, 29, 85, 27, -35, -64, 0, 0, 1, 0, 0, -64], family="t", qualifiers=[[17, -125]], values=[[0, 5, 18, 11]], timestamp=9223372036854775807, lockid=-1, durable=true, bufferable=true, attempt=2, region=RegionInfo(table="tsdb", region_name="tsdb,,1430141025970.93e30e6cb050703b91b3aae198c79e3c.", stop_key=[0, 0, 78, 85, 27, -105, 112, 0, 0, 1, 0, 0, -33, 0, 0, 13, 0, 0, 111, 0, 0, 14, 0, 0, 121, 0, 0, 15, 0, 0, 103, 0, 0, 16, 0, 0, 87])), PutRequest(table="tsdb", key=[0, 0, 33, 85, 27, -35, -64, 0, 0, 1, 0, 0, -64], family="t", qualifiers=[[17, -125]], values=["\x00\x06q""], timestamp=9223372036854775807, lockid=-1, durable=true, bufferable=true, attempt=2, region=RegionInfo(table="tsdb", region_name="tsdb,,1430141025970.93e30e6cb050703b91b3aae198c79e3c.", stop_key=[0, 0, 78, 85, 27, -105, 112, 0, 0, 1, 0, 0, -33, 0, 0, 13, 0, 0, 111, 0, 0, 14, 0, 0, 121, 0, 0, 15, 0, 0, 103, 0, 0, 16, 0, 0, 87])), ... 1020 RPCs not shown ..., PutRequest(table="tsdb", key=[0, 0, 31, 85, 28, 36, 16, 0, 0, 1, 0, 0, 33, 0, 0, 6, 0, 0, 40], family="t", qualifiers=[":\xC1"], values=[[1, -8]], timestamp=9223372036854775807, lockid=-1, durable=true, bufferable=true, attempt=2, region=RegionInfo(table="tsdb", region_name="tsdb,,1430141025970.93e30e6cb050703b91b3aae198c79e3c.", stop_key=[0, 0, 78, 85, 27, -105, 112, 0, 0, 1, 0, 0, -33, 0, 0, 13, 0, 0, 111, 0, 0, 14, 0, 0, 121, 0, 0, 15, 0, 0, 103, 0, 0, 16, 0, 0, 87]))]), rpcid=12874
2015-04-27 13:27:17,031 ERROR [New I/O worker #67] RegionClient: Unexpected exception from downstream on [id: 0x88d11ac2, /10.230.229.36:50555 => /10.230.229.49:60020]
java.lang.IndexOutOfBoundsException: Index: 1, Size: 1
at java.util.ArrayList.rangeCheck(ArrayList.java:635) ~[na:1.7.0_76]
at java.util.ArrayList.get(ArrayList.java:411) ~[na:1.7.0_76]
at java.util.Collections$UnmodifiableList.get(Collections.java:1211) ~[na:1.7.0_76]
at org.hbase.async.generated.ClientPB$MultiResponse.getRegionActionResult(ClientPB.java:18976) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.hbase.async.MultiAction.deserialize(MultiAction.java:536) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.hbase.async.RegionClient.decode(RegionClient.java:1334) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.hbase.async.RegionClient.decode(RegionClient.java:88) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:500) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435) ~[netty-3.9.1.Final.jar:na]
at org.hbase.async.RegionClient.handleUpstream(RegionClient.java:1093) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.hbase.async.HBaseClient$RegionClientPipeline.sendUpstream(HBaseClient.java:2655) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) ~[netty-3.9.1.Final.jar:na]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_76]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_76]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_76]
2015-04-27 13:22:17,746 ERROR [New I/O worker #57] RegionClient: Unexpected exception from downstream on [id: 0x798e3376, /10.230.229.36:50541 => /10.230.229.49:60020]
org.hbase.async.InvalidResponseException: Expected result #566 but got result #567
at org.hbase.async.MultiAction.deserialize(MultiAction.java:568) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.hbase.async.RegionClient.decode(RegionClient.java:1334) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.hbase.async.RegionClient.decode(RegionClient.java:88) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:500) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435) ~[netty-3.9.1.Final.jar:na]
at org.hbase.async.RegionClient.handleUpstream(RegionClient.java:1093) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.hbase.async.HBaseClient$RegionClientPipeline.sendUpstream(HBaseClient.java:2655) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) ~[netty-3.9.1.Final.jar:na]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_76]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_76]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_76]
2015-04-27 13:19:55,579 ERROR [New I/O worker #52] RegionClient: Unexpected exception from downstream on [id: 0xe1ed4d0f, /10.230.229.36:50536 :> /10.230.229.49:60020]
java.nio.channels.ClosedChannelException: null
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.cleanUpWriteBuffer(AbstractNioWorker.java:405) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.close(AbstractNioWorker.java:373) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.NioClientSocketPipelineSink.eventSunk(NioClientSocketPipelineSink.java:58) ~[netty-3.9.1.Final.jar:na]
at org.hbase.async.HBaseClient$RegionClientPipeline.sendDownstream(HBaseClient.java:2645) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.jboss.netty.channel.Channels.close(Channels.java:812) ~[netty-3.9.1.Final.jar:na]
at org.hbase.async.RegionClient.exceptionCaught(RegionClient.java:1109) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.hbase.async.RegionClient.handleUpstream(RegionClient.java:1093) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.hbase.async.HBaseClient$RegionClientPipeline.sendUpstream(HBaseClient.java:2655) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.jboss.netty.channel.Channels.fireExceptionCaught(Channels.java:525) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:48) ~[netty-3.9.1.Final.jar:na]
at org.hbase.async.HBaseClient$RegionClientPipeline.sendUpstream(HBaseClient.java:2655) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) ~[netty-3.9.1.Final.jar:na]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_76]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_76]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_76]
2015-04-27 13:27:17,038 WARN [4,queue=4,port=60020] ipc.RpcServer - RpcServer.respondercallId: 12873 service: ClientService methodName: Multi size: 58.8 K connection: 10.230.229.36:50555: output error
2015-04-27 13:27:17,039 WARN [4,queue=4,port=60020] ipc.RpcServer - B.DefaultRpcServer.handler=34,queue=4,port=60020: caught a ClosedChannelException, this means that the server isthbase01-dnds3-3-crd.eng.sfdc.net/10.230.229.49:60020 was processing a request but the client went away. The error message was: null
2015-04-27 13:27:17,064 WARN [.reader=8,port=60020] ipc.RpcServer - RpcServer.listener,port=60020: count of bytes read: 0
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
at sun.nio.ch.IOUtil.read(IOUtil.java:197)
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379)
at org.apache.hadoop.hbase.ipc.RpcServer.channelRead(RpcServer.java:2246)
at org.apache.hadoop.hbase.ipc.RpcServer$Connection.readAndProcess(RpcServer.java:1423)
at org.apache.hadoop.hbase.ipc.RpcServer$Listener.doRead(RpcServer.java:798)
at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:589)
at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:564)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2015-04-27 13:32:17,556 WARN [New I/O worker #85] HBaseClient: Probe Exists(table="tsdb", key=[0, 0, 79, 85, 30, 58, 112, 0, 0, 1, 0, 0, 33], family=null, qualifiers=null, attempt=0, region=RegionInfo(table="tsdb", region_name="tsdb,\x00\x00NU\x1B\x97p\x00\x00\x01\x00\x00\xDF\x00\x00\x0D\x00\x00o\x00\x00\x0E\x00\x00y\x00\x00\x0F\x00\x00g\x00\x00\x10\x00\x00W,1430141397269.0a544d28022eaaaabfee319f3a77f582.", stop_key=[0, 0, -54, 85, 28, -108, -112, 0, 0, 1, 0, 0, 1, 0, 0, 8, 0, 0, 54])) failed
org.hbase.async.RemoteException: org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=isthbase01-dnds4-3-crd.eng.sfdc.net port=60020 startCode=1430135168621. As of locationSeqNum=402070.
at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2787)
at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)
at org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2928)
at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30804)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
at java.lang.Thread.run(Thread.java:745)
at org.hbase.async.RegionClient.makeException(RegionClient.java:1473) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.hbase.async.RegionClient.decodeException(RegionClient.java:1491) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.hbase.async.RegionClient.decode(RegionClient.java:1322) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.hbase.async.RegionClient.decode(RegionClient.java:88) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:500) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) [netty-3.9.1.Final.jar:na]
at org.hbase.async.RegionClient.handleUpstream(RegionClient.java:1093) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) [netty-3.9.1.Final.jar:na]
at org.hbase.async.HBaseClient$RegionClientPipeline.sendUpstream(HBaseClient.java:2655) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) [netty-3.9.1.Final.jar:na]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_76]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_76]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_76]
2015-04-27 13:23:55,649 ERROR [New I/O worker #70] RegionClient: After decoding the last message on [id: 0x92afc6d3, /10.230.229.36:35772 :> /10.230.229.44:60020], there was still some undecoded bytes in the channel's buffer (which are going to be lost): ReplayingDecoderBuffer(ridx=60479, widx=61184)="\x00\x00\x03\xFD\xFB\x07\x08\xD0\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD1\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD2\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD3\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD5\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD4\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD6\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD7\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD8\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD9\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xDA\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xDB\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xDC\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xDD\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xDE\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xDF\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE0\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE1\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE2\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE3\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE4\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE5\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE6\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE8\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE7\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE9\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xEB\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xEA\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xEC\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xED\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xEE\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xEF\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF0\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF1\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF3\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF2\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF4\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF5\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF6\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF7\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF8\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF9\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xFA\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xFB\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xFC\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xFD\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xFE\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xFF\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x80\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x82\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x81\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x83\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x84\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x85\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x87\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x86\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x88\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x89\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x8A\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x8C\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache."
2015-04-27 13:19:56,245 ERROR [New I/O worker #53] RegionClient: After decoding the last message on [id: 0x9d437a85, /10.230.229.36:50537 :> /10.230.229.49:60020], there was still some undecoded bytes in the channel's buffer (which are going to be lost): ReplayingDecoderBuffer(ridx=12770, widx=13318)="\x00\x00\x03\xD2\xD0\x07\x08\xB9\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xBB\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xBC\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xBD\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xBE\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xBF\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xC0\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xC1\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xC2\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xC3\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xC4\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xC5\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xC6\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xC7\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop"
2015-04-27 13:19:56,245 WARN [RpcServer.responder] ipc.RpcServer - RpcServer.respondercallId: 1771 service: ClientService methodName: Mutate size: 142 connection: 10.230.229.36:50537: output error
2015-04-27 13:19:56,245 INFO [RpcServer.responder] ipc.RpcServer - RpcServer.responder: asyncWrite
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.writev0(Native Method)
at sun.nio.ch.SocketDispatcher.writev(SocketDispatcher.java:51)
at sun.nio.ch.IOUtil.write(IOUtil.java:148)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:524)
at org.apache.hadoop.hbase.ipc.BufferChain.write(BufferChain.java:106)
at org.apache.hadoop.hbase.ipc.RpcServer.channelWrite(RpcServer.java:2226)
at org.apache.hadoop.hbase.ipc.RpcServer$Responder.processResponse(RpcServer.java:1012)
at org.apache.hadoop.hbase.ipc.RpcServer$Responder.doAsyncWrite(RpcServer.java:952)
at org.apache.hadoop.hbase.ipc.RpcServer$Responder.doRunLoop(RpcServer.java:881)
at org.apache.hadoop.hbase.ipc.RpcServer$Responder.run(RpcServer.java:857)
2015-04-27 13:27:17,018 WARN [New I/O worker #67] HBaseClient: Probe Exists(table="tsdb", key=[0, 0, 22, 85, 27, -35, -64, 0, 0, 1, 0, 0, -64], family=null, qualifiers=null, attempt=0, region=RegionInfo(table="tsdb", region_name="tsdb,,1430141025970.93e30e6cb050703b91b3a
ae198c79e3c.", stop_key=[0, 0, 78, 85, 27, -105, 112, 0, 0, 1, 0, 0, -33, 0, 0, 13, 0, 0, 111, 0, 0, 14, 0, 0, 121, 0, 0, 15, 0, 0, 103, 0, 0, 16, 0, 0, 87])) failed
org.hbase.async.RemoteException: org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=isthbase01-dnds4-2-crd.eng.sfdc.net port=60020 startCode=1430135168598. As of locationSeqNum=305417.
at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2787)
at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)
at org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2928)
at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30804)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
at java.lang.Thread.run(Thread.java:745)
at org.hbase.async.RegionClient.makeException(RegionClient.java:1473) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.hbase.async.RegionClient.decodeException(RegionClient.java:1491) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.hbase.async.RegionClient.decode(RegionClient.java:1322) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.hbase.async.RegionClient.decode(RegionClient.java:88) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:500) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) [netty-3.9.1.Final.jar:na]
at org.hbase.async.RegionClient.handleUpstream(RegionClient.java:1093) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) [netty-3.9.1.Final.jar:na]
at org.hbase.async.HBaseClient$RegionClientPipeline.sendUpstream(HBaseClient.java:2655) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0]
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) [netty-3.9.1.Final.jar:na]
at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) [netty-3.9.1.Final.jar:na]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_76]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_76]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_76]
2015-04-28 05:50:40,714 ERROR [New I/O worker #33] ConnectionManager: Unexpected exception from downstream for [id: 0x4a556bde, /10.230.229.41:50696 => /10.230.229.36:4242]
java.io.IOException: Broken pipe
at sun.nio.ch.FileChannelImpl.transferTo0(Native Method) ~[na:1.7.0_76]
at sun.nio.ch.FileChannelImpl.transferToDirectly(FileChannelImpl.java:433) ~[na:1.7.0_76]
at sun.nio.ch.FileChannelImpl.transferTo(FileChannelImpl.java:565) ~[na:1.7.0_76]
at org.jboss.netty.channel.DefaultFileRegion.transferTo(DefaultFileRegion.java:68) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.SocketSendBufferPool$FileSendBuffer.transferTo(SocketSendBufferPool.java:331) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.write0(AbstractNioWorker.java:201) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.writeFromUserCode(AbstractNioWorker.java:146) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.NioServerSocketPipelineSink.handleAcceptedSocket(NioServerSocketPipelineSink.java:99) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.NioServerSocketPipelineSink.eventSunk(NioServerSocketPipelineSink.java:36) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:60) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.Channels.write(Channels.java:704) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.Channels.write(Channels.java:671) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.AbstractChannel.write(AbstractChannel.java:248) ~[netty-3.9.1.Final.jar:na]
at net.opentsdb.tsd.HttpQuery.sendFile(HttpQuery.java:949) ~[opentsdb-2.0.0-sfdc-1.0.4.jar:2.0.0-sfdc-1.0.4]
at net.opentsdb.tsd.HttpQuery.sendFile(HttpQuery.java:894) ~[opentsdb-2.0.0-sfdc-1.0.4.jar:2.0.0-sfdc-1.0.4]
at net.opentsdb.tsd.GraphHandler.respondAsciiQuery(GraphHandler.java:818) ~[opentsdb-2.0.0-sfdc-1.0.4.jar:2.0.0-sfdc-1.0.4]
at net.opentsdb.tsd.GraphHandler.doGraph(GraphHandler.java:214) ~[opentsdb-2.0.0-sfdc-1.0.4.jar:2.0.0-sfdc-1.0.4]
at net.opentsdb.tsd.GraphHandler.execute(GraphHandler.java:121) ~[opentsdb-2.0.0-sfdc-1.0.4.jar:2.0.0-sfdc-1.0.4]
at net.opentsdb.tsd.RpcHandler.handleHttpQuery(RpcHandler.java:255) ~[opentsdb-2.0.0-sfdc-1.0.4.jar:2.0.0-sfdc-1.0.4]
at net.opentsdb.tsd.RpcHandler.messageReceived(RpcHandler.java:163) ~[opentsdb-2.0.0-sfdc-1.0.4.jar:2.0.0-sfdc-1.0.4]
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:459) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:536) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler.handleUpstream(IdleStateAwareChannelHandler.java:36) ~[netty-3.9.1.Final.jar:na]
at net.opentsdb.tsd.ConnectionManager.handleUpstream(ConnectionManager.java:90) ~[opentsdb-2.0.0-sfdc-1.0.4.jar:2.0.0-sfdc-1.0.4]
at org.jboss.netty.handler.timeout.IdleStateHandler.messageReceived(IdleStateHandler.java:294) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) ~[netty-3.9.1.Final.jar:na]
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) ~[netty-3.9.1.Final.jar:na]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_76]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_76]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_76]
2015-04-27 18:41:55,966 ERROR [New I/O worker #73] RegionClient: Uncaught error during de-serialization of MultiAction(batch=[PutRequest(table="tsdb", key=[0, 2, 114, 85, 34, 46, -16, 0, 0, 1, 0, 0, -64, 0, 0, 4, 0, 0, 66, 0, 0, 5, 0, 0, -23], family="t", qualifiers=["\xA
0W"], values=[[0, 0, 0, 0, -101, -56, 35, 75]], timestamp=9223372036854775807, lockid=-1, durable=true, bufferable=true, attempt=2, region=RegionInfo(table="tsdb", region_name="tsdb,\x00\x02rU\x1B\xA5\x80\x00\x00\x01\x00\x00\xC6\x00\x00\x04\x00\x00?\x00\x00\x05\x00\x00\xEB,1430145994217.d2d866be2678f18d0a193d697a53bea1.", stop_key=[0, 2, 118, 85, 32, 108, -16, 0, 0, 1, 0, 0, -61, 0, 0, 3, 0, 0, 20])), PutRequest(table="tsdb", key=[0, 2, 114, 85, 34, 46, -16, 0, 0, 1, 0, 0, -64, 0, 0, 4, 0, 0, 80, 0, 0, 5, 0, 0, -24], family="t", qualifiers=["\xA0W"], values=[[0, 0, 0, 0, -54, -41, 54, 23]], timestamp=9223372036854775807, lockid=-1, durable=true, bufferable=true, attempt=1, region=RegionInfo(table="tsdb", region_name="tsdb,\x00\x02rU\x1B\xA5\x80\x00\x00\x01\x00\x00\xC6\x00\x00\x04\x00\x00?\x00\x00\x05\x00\x00\xEB,1430145994217.d2d866be2678f18d0a193d697a53bea1.", stop_key=[0, 2, 118, 85, 32, 108, -16, 0, 0, 1, 0, 0, -61, 0, 0, 3, 0, 0, 20])), ... 1021 RPCs not shown ..., PutRequest(table="tsdb", key=[0, 2, 118, 85, 59, 73, -128, 0, 0, 1, 0, 0, -63, 0, 0, 3, 0, 0, -78], family="t", qualifiers=[[-121, 3]], values=[[0, -110, 0, -84]], timestamp=9223372036854775807, lockid=-1, durable=true, bufferable=true, attempt=1, region=RegionInfo(table="tsdb", region_name="tsdb,\x00\x02vU l\xF0\x00\x00\x01\x00\x00\xC3\x00\x00\x03\x00\x00\x14,1430145994217
.4b08188eabf5b4fe92d3a0b0f8a3fc5b.", stop_key=[0, 2, -81, 85, 31, 97, -64, 0, 0, 1, 0, 0, -28]))]), rpcid=1208
2015-04-28 18:07:57,521 WARN [New I/O worker #51] uid.UniqueId - Race condition: tried to assign ID 338 to metrics:yarn.resourcemanager.running_0, but CAS failed on PutRequest(table="tsdb-uid", key="yarn.resourcemanager.running_0", family="id", qualifiers=["metrics"], va
lues=["\x00\x01R"], timestamp=9223372036854775807, lockid=-1, durable=true, bufferable=true, attempt=0, region=null), which indicates this UID must have been allocated concurrently by another TSD or thread. So ID 338 was leaked.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.