Created
June 10, 2013 15:23
-
-
Save aochsner/5749640 to your computer and use it in GitHub Desktop.
This file has been truncated, but you can view the full file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [2013-06-07 18:13:29,726][INFO ][discovery.zen ] [sthapqa01] master_left [[sthapqa02][-q1i2VXhTh6vbYOhQXCtOg][inet[/10.28.130.57:9300]]], reason [failed to ping, tried [3] times, each with maximum [30s] timeout] | |
| [2013-06-07 18:13:30,094][INFO ][cluster.service ] [sthapqa01] master {new [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]], previous [sthapqa02][-q1i2VXhTh6vbYOhQXCtOg][inet[/10.28.130.57:9300]]}, removed {[sthapqa02][-q1i2VXhTh6vbYOhQXCtOg][inet[/10.28.130.57:9300]],}, reason: zen-disco-master_failed ([sthapqa02][-q1i2VXhTh6vbYOhQXCtOg][inet[/10.28.130.57:9300]]) | |
| [2013-06-07 18:16:19,800][WARN ][transport ] [sthapqa01] Received response for a request that has timed out, sent [79454ms] ago, timed out [49454ms] ago, action [/cluster/nodes/indices/shard/store/n], node [[sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]]], id [2133054] | |
| [2013-06-07 18:16:20,817][WARN ][transport ] [sthapqa01] Received response for a request that has timed out, sent [140488ms] ago, timed out [110488ms] ago, action [/cluster/nodes/indices/shard/store/n], node [[sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]]], id [2133052] | |
| [2013-06-07 18:16:30,865][WARN ][transport ] [sthapqa01] Received response for a request that has timed out, sent [120521ms] ago, timed out [90520ms] ago, action [/cluster/nodes/indices/shard/store/n], node [[sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]]], id [2133053] | |
| [2013-06-07 18:16:32,273][WARN ][transport ] [sthapqa01] Received response for a request that has timed out, sent [61928ms] ago, timed out [31928ms] ago, action [/cluster/nodes/indices/shard/store/n], node [[sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]]], id [2133055] | |
| [2013-06-07 18:16:59,793][WARN ][transport ] [sthapqa01] Received response for a request that has timed out, sent [209509ms] ago, timed out [179509ms] ago, action [/cluster/nodes/indices/shard/store/n], node [[sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]]], id [2133051] | |
| [2013-06-07 18:16:59,793][WARN ][transport ] [sthapqa01] Received response for a request that has timed out, sent [49611ms] ago, timed out [19611ms] ago, action [/cluster/nodes/indices/shard/store/n], node [[sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]]], id [2133057] | |
| [2013-06-07 18:17:03,542][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:03,542][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:03,696][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,697][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,710][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,710][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,754][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:03,760][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,760][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,761][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:03,792][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,792][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,810][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:03,817][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,817][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,818][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:03,819][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,819][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,827][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:03,833][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,833][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,980][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:03,980][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,980][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,981][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:03,981][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,981][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,990][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:03,995][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,995][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,995][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:03,996][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:03,996][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,340][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,394][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,394][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,395][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,395][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,395][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,404][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,408][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,409][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,410][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,410][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,410][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,418][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,424][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,424][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,424][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,424][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,424][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,433][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,439][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,439][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,440][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,440][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,440][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,460][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,465][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,465][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,466][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,466][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,466][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,475][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,481][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,481][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,481][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,481][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,482][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,518][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,523][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,523][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,524][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,524][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,524][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,532][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,538][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,538][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,538][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,538][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,538][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,547][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,552][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,552][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,553][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,553][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,553][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,561][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,567][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,567][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,567][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,568][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,568][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,712][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,717][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,717][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,717][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,718][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,718][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,726][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,732][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,732][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,733][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,733][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,733][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,782][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,787][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,787][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,787][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,788][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,788][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,795][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,800][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,800][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,800][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,801][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,801][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,808][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,813][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,813][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,814][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,814][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,814][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,822][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,827][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,827][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,828][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,828][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,828][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,836][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,840][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,840][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,841][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,841][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,841][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,849][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,853][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,853][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,854][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,854][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,854][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,862][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,867][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,867][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,867][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,868][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,868][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,876][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,882][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,882][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,882][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,883][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,883][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,890][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,896][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,896][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,896][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,897][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,897][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,905][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,910][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,910][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,911][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,911][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,911][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,919][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,924][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,924][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,925][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,925][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,925][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,933][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,938][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,938][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,938][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,938][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,938][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,947][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,952][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,952][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,953][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,953][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,953][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,960][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,966][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,966][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,967][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,967][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,967][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,975][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,981][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,981][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,982][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,982][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,982][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,989][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,995][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,995][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,995][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:04,995][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:04,995][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,006][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,010][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,010][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,011][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,011][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,011][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,019][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,023][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,023][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,024][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,024][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,024][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,032][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,037][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,037][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,038][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,038][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,039][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,047][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,053][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,053][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,054][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,054][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,054][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,104][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,110][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,110][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,110][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,110][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,110][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,170][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,178][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,178][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,179][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,180][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,180][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,188][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,193][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,193][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,194][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,194][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,194][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,203][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,207][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,207][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,208][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,209][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,209][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,220][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,226][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,226][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,227][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,227][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,227][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,257][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,262][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,262][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,263][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,263][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,263][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,272][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,276][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,276][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,276][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,276][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,276][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,284][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,289][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,289][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,290][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,290][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,290][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,297][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,302][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,302][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,302][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,303][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,303][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,310][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,315][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,315][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,315][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,315][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,315][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,323][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,328][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,328][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,328][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,329][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,329][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,336][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,340][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,340][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,341][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,341][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,341][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,348][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,354][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,354][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,355][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,355][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,355][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,362][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,366][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,366][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,367][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,367][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,367][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,375][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,380][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,380][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,380][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,380][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,381][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,388][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,393][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,393][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,394][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,394][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,394][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,402][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,407][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,407][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,408][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,408][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,408][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,416][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,421][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,421][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,422][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,422][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,422][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,429][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,435][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,435][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,435][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,435][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,436][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,442][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,447][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,447][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,448][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,448][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,448][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,455][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,459][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,459][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,460][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,460][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,460][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,467][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,472][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,472][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,473][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,473][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,473][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,481][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,485][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,485][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,486][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,486][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,486][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,494][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,499][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,499][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,499][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,499][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,500][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,507][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,511][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,511][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,511][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,512][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,512][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,519][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,524][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,524][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,525][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,525][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,525][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,532][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,536][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,536][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,537][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,537][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,537][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,544][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,549][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,549][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,550][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,550][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,550][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,591][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,597][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,597][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,597][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,598][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,598][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,606][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,611][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,611][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,612][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,612][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,612][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,620][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,626][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,626][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,627][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,627][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,627][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,635][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,641][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,641][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,641][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,641][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,641][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,649][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,655][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,655][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,655][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,655][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,655][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,664][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,670][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,670][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,670][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,671][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,671][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,678][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,683][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,683][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,684][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,684][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,684][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,691][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,697][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,697][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,697][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,698][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,698][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,705][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,710][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,710][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.01][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,711][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,711][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,711][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,718][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.05.31][1] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,723][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,723][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.05.31][1], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.05.31][1]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,723][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.02][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:107) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:312) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: java.io.EOFException | |
| at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577) | |
| at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1923) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1897) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1970) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1894) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1777) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:169) | |
| ... 23 more | |
| [2013-06-07 18:17:05,724][WARN ][cluster.action.shard ] [sthapqa01] sending failed shard for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,724][WARN ][cluster.action.shard ] [sthapqa01] received shard failed for [stathub-qaint-activity-2013.06.02][2], node[FbPlzdQcT76X_1XxOu2P4w], [R], s[INITIALIZING], reason [Failed to start shard, message [RecoveryFailedException[[stathub-qaint-activity-2013.06.02][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]]]; nested: RemoteTransportException[Failed to deserialize exception response from stream]; nested: TransportSerializationException[Failed to deserialize exception response from stream]; nested: EOFException; ]] | |
| [2013-06-07 18:17:05,731][WARN ][indices.cluster ] [sthapqa01] [stathub-qaint-activity-2013.06.01][2] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [stathub-qaint-activity-2013.06.01][2]: Recovery failed from [sthapqa03][sM-IHqI8T3u9KlZgsPF-Ag][inet[/10.28.130.58:9300]] into [sthapqa01][FbPlzdQcT76X_1XxOu2P4w][inet[/10.28.130.56:9300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:293) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:62) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:163) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:722) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: Failed to deserialize exception response from stream | |
| Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize exception response from stream | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handlerResponseError(MessageChannelHandler.java:171) | |
| at org.elasticsearch.transport.netty.Messag |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment