Created
July 20, 2018 12:23
-
-
Save cdahlqvist/372471080255b444f42e3dba21fe3815 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
[2018-07-19T08:48:47,631][WARN ][o.e.i.c.IndicesClusterStateService] [iZBcaR9] [[elasticlogs-2-2018.01.02-44][1]] marking and sending shard failed due to [failed recovery] | |
org.elasticsearch.indices.recovery.RecoveryFailedException: [elasticlogs-2-2018.01.02-44][1]: Recovery failed from {Q8apBcz}{Q8apBcz2QeS7wXq3tIrvLA}{gY6uDjozSmWxadECk2VenQ}{192.168.1.32}{192.168.1.32:9300}{ml.machine_memory=128847142912, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true} into {iZBcaR9}{iZBcaR9JTk6f8OhyT7yL4A}{9v9sDmqZTniJCFvk_MSk6g}{192.168.1.33}{192.168.1.33:9300}{ml.machine_memory=128847142912, xpack.installed=true, ml.max_open_jobs=20, ml.enabled=true} | |
at org.elasticsearch.indices.recovery.PeerRecoveryTargetService.doRecovery(PeerRecoveryTargetService.java:282) [elasticsearch-6.3.0.jar:6.3.0] | |
at org.elasticsearch.indices.recovery.PeerRecoveryTargetService.access$900(PeerRecoveryTargetService.java:80) [elasticsearch-6.3.0.jar:6.3.0] | |
at org.elasticsearch.indices.recovery.PeerRecoveryTargetService$RecoveryRunner.doRun(PeerRecoveryTargetService.java:623) [elasticsearch-6.3.0.jar:6.3.0] | |
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:724) [elasticsearch-6.3.0.jar:6.3.0] | |
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.3.0.jar:6.3.0] | |
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1135) [?:?] | |
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?] | |
at java.lang.Thread.run(Thread.java:844) [?:?] | |
Caused by: org.elasticsearch.transport.RemoteTransportException: [Q8apBcz][172.18.0.2:9300][internal:index/shard/recovery/start_recovery] | |
Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [29432330242/27.4gb], which is larger than the limit of [28681371648/26.7gb] | |
at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:230) ~[elasticsearch-6.3.0.jar:6.3.0] | |
at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-6.3.0.jar:6.3.0] | |
at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1540) ~[elasticsearch-6.3.0.jar:6.3.0] | |
at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1414) ~[elasticsearch-6.3.0.jar:6.3.0] | |
at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:64) ~[?:?] | |
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) ~[?:?] | |
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) ~[?:?] | |
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) ~[?:?] | |
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) ~[?:?] | |
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] | |
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1336) ~[?:?] | |
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1127) ~[?:?] | |
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1162) ~[?:?] | |
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489) ~[?:?] | |
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428) ~[?:?] | |
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) ~[?:?] | |
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359) ~[?:?] | |
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935) ~[?:?] | |
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) ~[?:?] | |
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645) ~[?:?] | |
at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:545) ~[?:?] | |
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:499) ~[?:?] | |
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459) ~[?:?] | |
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) ~[?:?] | |
at java.lang.Thread.run(Thread.java:844) ~[?:?] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment