Skip to content

Instantly share code, notes, and snippets.

@carasel
Created October 11, 2018 14:13
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save carasel/34467b214bd71e9680da3ffa4d8584b2 to your computer and use it in GitHub Desktop.
Save carasel/34467b214bd71e9680da3ffa4d8584b2 to your computer and use it in GitHub Desktop.
::: {-EmC53_}{-EmC53_CRiC64VMXmAtFiQ}{7-FdaraSSH63rptu1Pc8cQ}{x.x.x.x}{x.x.x.x:9300}{zone=eu-west-1a}
Hot threads at 2018-10-11T13:51:16.637Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
::: {mGhV-nU}{mGhV-nUzTja_yet5D5AwAg}{0IA4BoVRTUul0mRVJFVtGA}{x.x.x.x}{x.x.x.x:9300}{zone=eu-west-1a, }
Hot threads at 2018-10-11T13:51:16.628Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
27.9% (139.3ms out of 500ms) cpu usage by thread 'Log4j2-TF-2-RollingFileManager-809'
9/10 snapshots sharing following 14 elements
java.util.zip.Deflater.deflateBytes(Native Method)
java.util.zip.Deflater.deflate(Deflater.java:444)
java.util.zip.Deflater.deflate(Deflater.java:366)
java.util.zip.DeflaterOutputStream.deflate(DeflaterOutputStream.java:251)
java.util.zip.DeflaterOutputStream.write(DeflaterOutputStream.java:211)
java.util.zip.GZIPOutputStream.write(GZIPOutputStream.java:145)
java.io.BufferedOutputStream.write(BufferedOutputStream.java:122)
org.apache.logging.log4j.core.appender.rolling.action.GzCompressAction.execute(GzCompressAction.java:97)
org.apache.logging.log4j.core.appender.rolling.action.GzCompressAction.execute(GzCompressAction.java:74)
org.apache.logging.log4j.core.appender.rolling.RollingFileManager$AsyncAction.execute(RollingFileManager.java:459)
org.apache.logging.log4j.core.appender.rolling.action.AbstractAction.run(AbstractAction.java:66)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
unique snapshot
java.io.FileOutputStream.writeBytes(Native Method)
java.io.FileOutputStream.write(FileOutputStream.java:326)
java.util.zip.DeflaterOutputStream.deflate(DeflaterOutputStream.java:253)
java.util.zip.DeflaterOutputStream.write(DeflaterOutputStream.java:211)
java.util.zip.GZIPOutputStream.write(GZIPOutputStream.java:145)
java.io.BufferedOutputStream.write(BufferedOutputStream.java:122)
org.apache.logging.log4j.core.appender.rolling.action.GzCompressAction.execute(GzCompressAction.java:97)
org.apache.logging.log4j.core.appender.rolling.action.GzCompressAction.execute(GzCompressAction.java:74)
org.apache.logging.log4j.core.appender.rolling.RollingFileManager$AsyncAction.execute(RollingFileManager.java:459)
org.apache.logging.log4j.core.appender.rolling.action.AbstractAction.run(AbstractAction.java:66)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
16.5% (82.2ms out of 500ms) cpu usage by thread 'elasticsearch[mGhV-nU][transport_client_boss][T#8]'
3/10 snapshots sharing following 66 elements
org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)
org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)
org.apache.logging.log4j.core.appender.RollingFileAppender.append(RollingFileAppender.java:308)
org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)
org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)
org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)
org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:63)
org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
org.apache.logging.log4j.spi.ExtendedLoggerWrapper.logMessage(ExtendedLoggerWrapper.java:217)
org.elasticsearch.common.logging.PrefixLogger.logMessage(PrefixLogger.java:102)
org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2116)
org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2100)
org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:1989)
org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1851)
org.apache.logging.log4j.spi.AbstractLogger.debug(AbstractLogger.java:337)
org.elasticsearch.action.search.InitialSearchPhase.onShardFailure(InitialSearchPhase.java:126)
org.elasticsearch.action.search.InitialSearchPhase.access$100(InitialSearchPhase.java:49)
org.elasticsearch.action.search.InitialSearchPhase$2.lambda$onFailure$1(InitialSearchPhase.java:217)
org.elasticsearch.action.search.InitialSearchPhase$2$$Lambda$2243/2068475858.run(Unknown Source)
org.elasticsearch.action.search.InitialSearchPhase.maybeFork(InitialSearchPhase.java:171)
org.elasticsearch.action.search.InitialSearchPhase.access$000(InitialSearchPhase.java:49)
org.elasticsearch.action.search.InitialSearchPhase$2.onFailure(InitialSearchPhase.java:217)
org.elasticsearch.action.search.SearchExecutionStatsCollector.onFailure(SearchExecutionStatsCollector.java:73)
org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:51)
org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:527)
org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1098)
org.elasticsearch.transport.TcpTransport.lambda$handleException$33(TcpTransport.java:1478)
org.elasticsearch.transport.TcpTransport$$Lambda$1926/1128593633.run(Unknown Source)
org.elasticsearch.common.util.concurrent.EsExecutors$1.execute(EsExecutors.java:135)
org.elasticsearch.transport.TcpTransport.handleException(TcpTransport.java:1476)
org.elasticsearch.transport.TcpTransport.handlerResponseError(TcpTransport.java:1468)
org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1398)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:64)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:545)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:499)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
java.lang.Thread.run(Thread.java:748)
3/10 snapshots sharing following 19 elements
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:545)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:499)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
java.lang.Thread.run(Thread.java:748)
4/10 snapshots sharing following 2 elements
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
java.lang.Thread.run(Thread.java:748)
16.0% (80.2ms out of 500ms) cpu usage by thread 'elasticsearch[mGhV-nU][transport_client_boss][T#7]'
3/10 snapshots sharing following 65 elements
org.apache.logging.log4j.core.appender.rolling.RollingFileManager.checkRollover(RollingFileManager.java:251)
org.apache.logging.log4j.core.appender.RollingFileAppender.append(RollingFileAppender.java:307)
org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)
org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)
org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)
org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:63)
org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
org.apache.logging.log4j.spi.ExtendedLoggerWrapper.logMessage(ExtendedLoggerWrapper.java:217)
org.elasticsearch.common.logging.PrefixLogger.logMessage(PrefixLogger.java:102)
org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2116)
org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2100)
org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:1989)
org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1851)
org.apache.logging.log4j.spi.AbstractLogger.debug(AbstractLogger.java:337)
org.elasticsearch.action.search.InitialSearchPhase.onShardFailure(InitialSearchPhase.java:126)
org.elasticsearch.action.search.InitialSearchPhase.access$100(InitialSearchPhase.java:49)
org.elasticsearch.action.search.InitialSearchPhase$2.lambda$onFailure$1(InitialSearchPhase.java:217)
org.elasticsearch.action.search.InitialSearchPhase$2$$Lambda$2243/2068475858.run(Unknown Source)
org.elasticsearch.action.search.InitialSearchPhase.maybeFork(InitialSearchPhase.java:171)
org.elasticsearch.action.search.InitialSearchPhase.access$000(InitialSearchPhase.java:49)
org.elasticsearch.action.search.InitialSearchPhase$2.onFailure(InitialSearchPhase.java:217)
org.elasticsearch.action.search.SearchExecutionStatsCollector.onFailure(SearchExecutionStatsCollector.java:73)
org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:51)
org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:527)
org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1098)
org.elasticsearch.transport.TcpTransport.lambda$handleException$33(TcpTransport.java:1478)
org.elasticsearch.transport.TcpTransport$$Lambda$1926/1128593633.run(Unknown Source)
org.elasticsearch.common.util.concurrent.EsExecutors$1.execute(EsExecutors.java:135)
org.elasticsearch.transport.TcpTransport.handleException(TcpTransport.java:1476)
org.elasticsearch.transport.TcpTransport.handlerResponseError(TcpTransport.java:1468)
org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1398)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:64)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:545)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:499)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
java.lang.Thread.run(Thread.java:748)
2/10 snapshots sharing following 75 elements
org.apache.logging.log4j.message.ParameterizedMessage.formatTo(ParameterizedMessage.java:228)
org.apache.logging.log4j.core.pattern.MessagePatternConverter.format(MessagePatternConverter.java:119)
org.apache.logging.log4j.core.pattern.PatternFormatter.formatWithInfo(PatternFormatter.java:45)
org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:40)
org.apache.logging.log4j.core.layout.PatternLayout$PatternSerializer.toSerializable(PatternLayout.java:333)
org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:232)
org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:217)
org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:57)
org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:177)
org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)
org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)
org.apache.logging.log4j.core.appender.RollingFileAppender.append(RollingFileAppender.java:308)
org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)
org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)
org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)
org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:63)
org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
org.apache.logging.log4j.spi.ExtendedLoggerWrapper.logMessage(ExtendedLoggerWrapper.java:217)
org.elasticsearch.common.logging.PrefixLogger.logMessage(PrefixLogger.java:102)
org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2116)
org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2100)
org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:1989)
org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1851)
org.apache.logging.log4j.spi.AbstractLogger.debug(AbstractLogger.java:337)
org.elasticsearch.action.search.InitialSearchPhase.onShardFailure(InitialSearchPhase.java:126)
org.elasticsearch.action.search.InitialSearchPhase.access$100(InitialSearchPhase.java:49)
org.elasticsearch.action.search.InitialSearchPhase$2.lambda$onFailure$1(InitialSearchPhase.java:217)
org.elasticsearch.action.search.InitialSearchPhase$2$$Lambda$2243/2068475858.run(Unknown Source)
org.elasticsearch.action.search.InitialSearchPhase.maybeFork(InitialSearchPhase.java:171)
org.elasticsearch.action.search.InitialSearchPhase.access$000(InitialSearchPhase.java:49)
org.elasticsearch.action.search.InitialSearchPhase$2.onFailure(InitialSearchPhase.java:217)
org.elasticsearch.action.search.SearchExecutionStatsCollector.onFailure(SearchExecutionStatsCollector.java:73)
org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:51)
org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:527)
org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1098)
org.elasticsearch.transport.TcpTransport.lambda$handleException$33(TcpTransport.java:1478)
org.elasticsearch.transport.TcpTransport$$Lambda$1926/1128593633.run(Unknown Source)
org.elasticsearch.common.util.concurrent.EsExecutors$1.execute(EsExecutors.java:135)
org.elasticsearch.transport.TcpTransport.handleException(TcpTransport.java:1476)
org.elasticsearch.transport.TcpTransport.handlerResponseError(TcpTransport.java:1468)
org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1398)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:64)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:545)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:499)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
java.lang.Thread.run(Thread.java:748)
2/10 snapshots sharing following 73 elements
org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeChunkedText(TextEncoderHelper.java:146)
org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText(TextEncoderHelper.java:58)
org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:68)
org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32)
org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:219)
org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:57)
org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:177)
org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)
org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)
org.apache.logging.log4j.core.appender.RollingFileAppender.append(RollingFileAppender.java:308)
org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)
org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)
org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)
org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:63)
org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
org.apache.logging.log4j.spi.ExtendedLoggerWrapper.logMessage(ExtendedLoggerWrapper.java:217)
org.elasticsearch.common.logging.PrefixLogger.logMessage(PrefixLogger.java:102)
org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2116)
org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2100)
org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:1989)
org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1851)
org.apache.logging.log4j.spi.AbstractLogger.debug(AbstractLogger.java:337)
org.elasticsearch.action.search.InitialSearchPhase.onShardFailure(InitialSearchPhase.java:126)
org.elasticsearch.action.search.InitialSearchPhase.access$100(InitialSearchPhase.java:49)
org.elasticsearch.action.search.InitialSearchPhase$2.lambda$onFailure$1(InitialSearchPhase.java:217)
org.elasticsearch.action.search.InitialSearchPhase$2$$Lambda$2243/2068475858.run(Unknown Source)
org.elasticsearch.action.search.InitialSearchPhase.maybeFork(InitialSearchPhase.java:171)
org.elasticsearch.action.search.InitialSearchPhase.access$000(InitialSearchPhase.java:49)
org.elasticsearch.action.search.InitialSearchPhase$2.onFailure(InitialSearchPhase.java:217)
org.elasticsearch.action.search.SearchExecutionStatsCollector.onFailure(SearchExecutionStatsCollector.java:73)
org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:51)
org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:527)
org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1098)
org.elasticsearch.transport.TcpTransport.lambda$handleException$33(TcpTransport.java:1478)
org.elasticsearch.transport.TcpTransport$$Lambda$1926/1128593633.run(Unknown Source)
org.elasticsearch.common.util.concurrent.EsExecutors$1.execute(EsExecutors.java:135)
org.elasticsearch.transport.TcpTransport.handleException(TcpTransport.java:1476)
org.elasticsearch.transport.TcpTransport.handlerResponseError(TcpTransport.java:1468)
org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1398)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:64)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:545)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:499)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
java.lang.Thread.run(Thread.java:748)
2/10 snapshots sharing following 77 elements
org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:164)
org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:141)
org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:122)
org.apache.logging.log4j.core.impl.Log4jLogEvent.getThrownProxy(Log4jLogEvent.java:566)
org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter.format(ExtendedThrowablePatternConverter.java:64)
org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:38)
org.apache.logging.log4j.core.layout.PatternLayout$PatternSerializer.toSerializable(PatternLayout.java:333)
org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:232)
org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:217)
org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:57)
org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:177)
org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)
org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)
org.apache.logging.log4j.core.appender.RollingFileAppender.append(RollingFileAppender.java:308)
org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)
org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)
org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)
org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:63)
org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
org.apache.logging.log4j.spi.ExtendedLoggerWrapper.logMessage(ExtendedLoggerWrapper.java:217)
org.elasticsearch.common.logging.PrefixLogger.logMessage(PrefixLogger.java:102)
org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2116)
org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2100)
org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:1989)
org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1851)
org.apache.logging.log4j.spi.AbstractLogger.debug(AbstractLogger.java:337)
org.elasticsearch.action.search.InitialSearchPhase.onShardFailure(InitialSearchPhase.java:126)
org.elasticsearch.action.search.InitialSearchPhase.access$100(InitialSearchPhase.java:49)
org.elasticsearch.action.search.InitialSearchPhase$2.lambda$onFailure$1(InitialSearchPhase.java:217)
org.elasticsearch.action.search.InitialSearchPhase$2$$Lambda$2243/2068475858.run(Unknown Source)
org.elasticsearch.action.search.InitialSearchPhase.maybeFork(InitialSearchPhase.java:171)
org.elasticsearch.action.search.InitialSearchPhase.access$000(InitialSearchPhase.java:49)
org.elasticsearch.action.search.InitialSearchPhase$2.onFailure(InitialSearchPhase.java:217)
org.elasticsearch.action.search.SearchExecutionStatsCollector.onFailure(SearchExecutionStatsCollector.java:73)
org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:51)
org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:527)
org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1098)
org.elasticsearch.transport.TcpTransport.lambda$handleException$33(TcpTransport.java:1478)
org.elasticsearch.transport.TcpTransport$$Lambda$1926/1128593633.run(Unknown Source)
org.elasticsearch.common.util.concurrent.EsExecutors$1.execute(EsExecutors.java:135)
org.elasticsearch.transport.TcpTransport.handleException(TcpTransport.java:1476)
org.elasticsearch.transport.TcpTransport.handlerResponseError(TcpTransport.java:1468)
org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1398)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:64)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:545)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:499)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
java.lang.Thread.run(Thread.java:748)
unique snapshot
org.apache.lucene.util.CloseableThreadLocal.set(CloseableThreadLocal.java:97)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextThreadLocal.set(ThreadContext.java:515)
org.elasticsearch.common.util.concurrent.ThreadContext.lambda$newStoredContext$2(ThreadContext.java:140)
org.elasticsearch.common.util.concurrent.ThreadContext$$Lambda$1576/1706773689.close(Unknown Source)
org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.$closeResource(TransportService.java:1092)
org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1099)
org.elasticsearch.transport.TcpTransport.lambda$handleException$33(TcpTransport.java:1478)
org.elasticsearch.transport.TcpTransport$$Lambda$1926/1128593633.run(Unknown Source)
org.elasticsearch.common.util.concurrent.EsExecutors$1.execute(EsExecutors.java:135)
org.elasticsearch.transport.TcpTransport.handleException(TcpTransport.java:1476)
org.elasticsearch.transport.TcpTransport.handlerResponseError(TcpTransport.java:1468)
org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1398)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:64)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:545)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:499)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
java.lang.Thread.run(Thread.java:748)
::: {1MIihsl}{1MIihslwS-agDQiRXSE32w}{GgieNPA3Tyem3ggXB27wpg}{x.x.x.x}{x.x.x.x:9300}{zone=eu-west-1c}
Hot threads at 2018-10-11T13:51:16.919Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
::: {bLno577}{bLno577uSFGTljDIkoLyHQ}{uXTrF3OxQLCxErwl-lU1Ew}{x.x.x.x}{x.x.x.x:9300}{zone=eu-west-1a}
Hot threads at 2018-10-11T13:51:16.628Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
0.0% (137.3micros out of 500ms) cpu usage by thread 'MetricAggregationClientPublisher-1'
10/10 snapshots sharing following 14 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:522)
java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:684)
[AMAZON INTERNAL]
[AMAZON INTERNAL]
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
java.util.concurrent.FutureTask.run(FutureTask.java:266)
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
::: {NeCaeFk}{NeCaeFkRQridc977vYLaqQ}{Vwm-f3w1Q5uuCHuS90jkpg}{x.x.x.x}{x.x.x.x:9300}{zone=eu-west-1a}
Hot threads at 2018-10-11T13:51:16.628Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
29.3% (146.4ms out of 500ms) cpu usage by thread 'Log4j2-TF-2-RollingFileManager-803'
8/10 snapshots sharing following 14 elements
java.util.zip.Deflater.deflateBytes(Native Method)
java.util.zip.Deflater.deflate(Deflater.java:444)
java.util.zip.Deflater.deflate(Deflater.java:366)
java.util.zip.DeflaterOutputStream.deflate(DeflaterOutputStream.java:251)
java.util.zip.DeflaterOutputStream.write(DeflaterOutputStream.java:211)
java.util.zip.GZIPOutputStream.write(GZIPOutputStream.java:145)
java.io.BufferedOutputStream.write(BufferedOutputStream.java:122)
org.apache.logging.log4j.core.appender.rolling.action.GzCompressAction.execute(GzCompressAction.java:97)
org.apache.logging.log4j.core.appender.rolling.action.GzCompressAction.execute(GzCompressAction.java:74)
org.apache.logging.log4j.core.appender.rolling.RollingFileManager$AsyncAction.execute(RollingFileManager.java:459)
org.apache.logging.log4j.core.appender.rolling.action.AbstractAction.run(AbstractAction.java:66)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
2/10 snapshots sharing following 9 elements
java.io.FileInputStream.readBytes(Native Method)
java.io.FileInputStream.read(FileInputStream.java:233)
org.apache.logging.log4j.core.appender.rolling.action.GzCompressAction.execute(GzCompressAction.java:96)
org.apache.logging.log4j.core.appender.rolling.action.GzCompressAction.execute(GzCompressAction.java:74)
org.apache.logging.log4j.core.appender.rolling.RollingFileManager$AsyncAction.execute(RollingFileManager.java:459)
org.apache.logging.log4j.core.appender.rolling.action.AbstractAction.run(AbstractAction.java:66)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
25.7% (128.4ms out of 500ms) cpu usage by thread 'elasticsearch[NeCaeFk][search][T#6]'
2/10 snapshots sharing following 60 elements
java.lang.Throwable.fillInStackTrace(Native Method)
java.lang.Throwable.fillInStackTrace(Throwable.java:783)
java.lang.Throwable.<init>(Throwable.java:287)
java.lang.Exception.<init>(Exception.java:84)
java.lang.ReflectiveOperationException.<init>(ReflectiveOperationException.java:75)
java.lang.ClassNotFoundException.<init>(ClassNotFoundException.java:82)
java.net.URLClassLoader.findClass(URLClassLoader.java:381)
java.lang.ClassLoader.loadClass(ClassLoader.java:424)
sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
java.lang.ClassLoader.loadClass(ClassLoader.java:357)
org.apache.logging.log4j.core.util.Loader.loadClass(Loader.java:240)
org.apache.logging.log4j.core.impl.ThrowableProxy.loadClass(ThrowableProxy.java:593)
org.apache.logging.log4j.core.impl.ThrowableProxy.loadClass(ThrowableProxy.java:584)
org.apache.logging.log4j.core.impl.ThrowableProxy.toExtendedStackTrace(ThrowableProxy.java:698)
org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:164)
org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:141)
org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:122)
org.apache.logging.log4j.core.impl.Log4jLogEvent.getThrownProxy(Log4jLogEvent.java:566)
org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter.format(ExtendedThrowablePatternConverter.java:64)
org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:38)
org.apache.logging.log4j.core.layout.PatternLayout$PatternSerializer.toSerializable(PatternLayout.java:333)
org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:232)
org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:217)
org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:57)
org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:177)
org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)
org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)
org.apache.logging.log4j.core.appender.RollingFileAppender.append(RollingFileAppender.java:308)
org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)
org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)
org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)
org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:63)
org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
org.apache.logging.log4j.spi.ExtendedLoggerWrapper.logMessage(ExtendedLoggerWrapper.java:217)
org.elasticsearch.common.logging.PrefixLogger.logMessage(PrefixLogger.java:102)
org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2116)
org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2100)
org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:1989)
org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1851)
org.apache.logging.log4j.spi.AbstractLogger.debug(AbstractLogger.java:337)
org.elasticsearch.action.search.InitialSearchPhase.onShardFailure(InitialSearchPhase.java:96)
org.elasticsearch.action.search.InitialSearchPhase.access$100(InitialSearchPhase.java:49)
org.elasticsearch.action.search.InitialSearchPhase$2.lambda$onFailure$1(InitialSearchPhase.java:217)
org.elasticsearch.action.search.InitialSearchPhase$2$$Lambda$2253/565438834.run(Unknown Source)
org.elasticsearch.action.search.InitialSearchPhase$1.doRun(InitialSearchPhase.java:184)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
3/10 snapshots sharing following 30 elements
[AMAZON INTERNAL]
org.elasticsearch.rest.RestController$ResourceHandlingHttpChannel.sendResponse(RestController.java:491)
org.elasticsearch.rest.action.RestActionListener.onFailure(RestActionListener.java:58)
org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:91)
org.elasticsearch.action.search.AbstractSearchAsyncAction.raisePhaseFailure(AbstractSearchAsyncAction.java:222)
org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:274)
org.elasticsearch.action.search.FetchSearchPhase$1.onFailure(FetchSearchPhase.java:92)
org.elasticsearch.common.util.concurrent.AbstractRunnable.onRejection(AbstractRunnable.java:63)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.onRejection(ThreadContext.java:662)
org.elasticsearch.common.util.concurrent.TimedRunnable.onRejection(TimedRunnable.java:50)
org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.doExecute(EsThreadPoolExecutor.java:104)
org.elasticsearch.common.util.concurrent.QueueResizingEsThreadPoolExecutor.doExecute(QueueResizingEsThreadPoolExecutor.java:88)
org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.execute(EsThreadPoolExecutor.java:93)
org.elasticsearch.action.search.AbstractSearchAsyncAction.execute(AbstractSearchAsyncAction.java:289)
org.elasticsearch.action.search.FetchSearchPhase.run(FetchSearchPhase.java:81)
org.elasticsearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:146)
org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:140)
org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:243)
org.elasticsearch.action.search.InitialSearchPhase.onShardFailure(InitialSearchPhase.java:107)
org.elasticsearch.action.search.InitialSearchPhase.access$100(InitialSearchPhase.java:49)
org.elasticsearch.action.search.InitialSearchPhase$2.lambda$onFailure$1(InitialSearchPhase.java:217)
org.elasticsearch.action.search.InitialSearchPhase$2$$Lambda$2253/565438834.run(Unknown Source)
org.elasticsearch.action.search.InitialSearchPhase$1.doRun(InitialSearchPhase.java:184)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
2/10 snapshots sharing following 25 elements
org.apache.lucene.index.LeafReader.docFreq(LeafReader.java:82)
org.apache.lucene.index.BaseCompositeReader.docFreq(BaseCompositeReader.java:126)
org.elasticsearch.common.lucene.search.XMoreLikeThis.createQueue(XMoreLikeThis.java:740)
org.elasticsearch.common.lucene.search.XMoreLikeThis.createQueue(XMoreLikeThis.java:715)
org.elasticsearch.common.lucene.search.XMoreLikeThis.like(XMoreLikeThis.java:629)
org.elasticsearch.common.lucene.search.MoreLikeThisQuery.createQuery(MoreLikeThisQuery.java:172)
org.elasticsearch.common.lucene.search.MoreLikeThisQuery.rewrite(MoreLikeThisQuery.java:156)
org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:243)
org.apache.lucene.search.IndexSearcher.rewrite(IndexSearcher.java:675)
org.elasticsearch.search.internal.ContextIndexSearcher.rewrite(ContextIndexSearcher.java:106)
org.elasticsearch.search.DefaultSearchContext.preProcess(DefaultSearchContext.java:257)
org.elasticsearch.search.query.QueryPhase.preProcess(QueryPhase.java:89)
org.elasticsearch.search.SearchService.createContext(SearchService.java:570)
org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:534)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:330)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
3/10 snapshots sharing following 27 elements
org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.decodeMetaData(SegmentTermsEnumFrame.java:428)
org.apache.lucene.codecs.blocktree.SegmentTermsEnum.docFreq(SegmentTermsEnum.java:980)
org.apache.lucene.index.LeafReader.docFreq(LeafReader.java:83)
org.apache.lucene.index.BaseCompositeReader.docFreq(BaseCompositeReader.java:126)
org.elasticsearch.common.lucene.search.XMoreLikeThis.createQueue(XMoreLikeThis.java:740)
org.elasticsearch.common.lucene.search.XMoreLikeThis.createQueue(XMoreLikeThis.java:715)
org.elasticsearch.common.lucene.search.XMoreLikeThis.like(XMoreLikeThis.java:629)
org.elasticsearch.common.lucene.search.MoreLikeThisQuery.createQuery(MoreLikeThisQuery.java:172)
org.elasticsearch.common.lucene.search.MoreLikeThisQuery.rewrite(MoreLikeThisQuery.java:156)
org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:243)
org.apache.lucene.search.IndexSearcher.rewrite(IndexSearcher.java:675)
org.elasticsearch.search.internal.ContextIndexSearcher.rewrite(ContextIndexSearcher.java:106)
org.elasticsearch.search.DefaultSearchContext.preProcess(DefaultSearchContext.java:257)
org.elasticsearch.search.query.QueryPhase.preProcess(QueryPhase.java:89)
org.elasticsearch.search.SearchService.createContext(SearchService.java:570)
org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:534)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:330)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
25.0% (124.8ms out of 500ms) cpu usage by thread 'elasticsearch[NeCaeFk][transport_client_boss][T#6]'
5/10 snapshots sharing following 71 elements
java.util.regex.Pattern$Start.match(Pattern.java:3463)
java.util.regex.Matcher.search(Matcher.java:1248)
java.util.regex.Matcher.find(Matcher.java:637)
java.util.regex.Matcher.replaceAll(Matcher.java:951)
[AMAZON INTERNAL]
[AMAZON INTERNAL]
[AMAZON INTERNAL]
[AMAZON INTERNAL]
org.elasticsearch.rest.RestController$ResourceHandlingHttpChannel.sendResponse(RestController.java:491)
org.elasticsearch.rest.action.RestActionListener.onFailure(RestActionListener.java:58)
org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:91)
org.elasticsearch.action.search.AbstractSearchAsyncAction.raisePhaseFailure(AbstractSearchAsyncAction.java:222)
org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:274)
org.elasticsearch.action.search.FetchSearchPhase$1.onFailure(FetchSearchPhase.java:92)
org.elasticsearch.common.util.concurrent.AbstractRunnable.onRejection(AbstractRunnable.java:63)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.onRejection(ThreadContext.java:662)
org.elasticsearch.common.util.concurrent.TimedRunnable.onRejection(TimedRunnable.java:50)
org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.doExecute(EsThreadPoolExecutor.java:104)
org.elasticsearch.common.util.concurrent.QueueResizingEsThreadPoolExecutor.doExecute(QueueResizingEsThreadPoolExecutor.java:88)
org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.execute(EsThreadPoolExecutor.java:93)
org.elasticsearch.action.search.AbstractSearchAsyncAction.execute(AbstractSearchAsyncAction.java:289)
org.elasticsearch.action.search.FetchSearchPhase.run(FetchSearchPhase.java:81)
org.elasticsearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:146)
org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:140)
org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:243)
org.elasticsearch.action.search.InitialSearchPhase.successfulShardExecution(InitialSearchPhase.java:251)
org.elasticsearch.action.search.InitialSearchPhase.onShardResult(InitialSearchPhase.java:239)
org.elasticsearch.action.search.InitialSearchPhase.access$200(InitialSearchPhase.java:49)
org.elasticsearch.action.search.InitialSearchPhase$2.lambda$innerOnResponse$0(InitialSearchPhase.java:212)
org.elasticsearch.action.search.InitialSearchPhase$2$$Lambda$2026/2015229981.run(Unknown Source)
org.elasticsearch.action.search.InitialSearchPhase.maybeFork(InitialSearchPhase.java:171)
org.elasticsearch.action.search.InitialSearchPhase.access$000(InitialSearchPhase.java:49)
org.elasticsearch.action.search.InitialSearchPhase$2.innerOnResponse(InitialSearchPhase.java:212)
org.elasticsearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:45)
org.elasticsearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:29)
org.elasticsearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:68)
org.elasticsearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:36)
org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:46)
org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:518)
org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1091)
org.elasticsearch.transport.TcpTransport$2.doRun(TcpTransport.java:1452)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.EsExecutors$1.execute(EsExecutors.java:135)
org.elasticsearch.transport.TcpTransport.handleResponse(TcpTransport.java:1444)
org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1400)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:64)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:545)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:499)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
java.lang.Thread.run(Thread.java:748)
2/10 snapshots sharing following 70 elements
org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput.ensureCapacity(ReleasableBytesStreamOutput.java:69)
org.elasticsearch.common.io.stream.BytesStreamOutput.writeBytes(BytesStreamOutput.java:89)
org.elasticsearch.transport.CompressibleBytesOutputStream.writeBytes(CompressibleBytesOutputStream.java:86)
org.elasticsearch.common.io.stream.StreamOutput.writeBytes(StreamOutput.java:121)
org.elasticsearch.common.io.stream.StreamOutput.writeString(StreamOutput.java:344)
org.elasticsearch.common.io.stream.StreamOutput.writeStringArray(StreamOutput.java:414)
org.elasticsearch.index.query.MoreLikeThisQueryBuilder.doWriteTo(MoreLikeThisQueryBuilder.java:539)
org.elasticsearch.index.query.AbstractQueryBuilder.writeTo(AbstractQueryBuilder.java:73)
org.elasticsearch.common.io.stream.StreamOutput.writeNamedWriteable(StreamOutput.java:883)
org.elasticsearch.common.io.stream.StreamOutput.writeOptionalNamedWriteable(StreamOutput.java:894)
org.elasticsearch.search.builder.SearchSourceBuilder.writeTo(SearchSourceBuilder.java:252)
org.elasticsearch.common.io.stream.StreamOutput.writeOptionalWriteable(StreamOutput.java:758)
org.elasticsearch.search.internal.ShardSearchLocalRequest.innerWriteTo(ShardSearchLocalRequest.java:222)
org.elasticsearch.search.internal.ShardSearchTransportRequest.writeTo(ShardSearchTransportRequest.java:158)
org.elasticsearch.transport.TcpTransport.buildMessage(TcpTransport.java:1236)
org.elasticsearch.transport.TcpTransport.sendRequestToChannel(TcpTransport.java:1108)
org.elasticsearch.transport.TcpTransport.access$1700(TcpTransport.java:122)
org.elasticsearch.transport.TcpTransport$NodeChannels.sendRequest(TcpTransport.java:482)
org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:598)
org.elasticsearch.transport.TransportService$$Lambda$1243/179437727.sendRequest(Unknown Source)
org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:518)
org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:558)
org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:549)
org.elasticsearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:152)
org.elasticsearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:52)
org.elasticsearch.action.search.InitialSearchPhase.performPhaseOnShard(InitialSearchPhase.java:208)
org.elasticsearch.action.search.InitialSearchPhase.maybeExecuteNext(InitialSearchPhase.java:162)
org.elasticsearch.action.search.InitialSearchPhase.onShardFailure(InitialSearchPhase.java:122)
org.elasticsearch.action.search.InitialSearchPhase.access$100(InitialSearchPhase.java:49)
org.elasticsearch.action.search.InitialSearchPhase$2.lambda$onFailure$1(InitialSearchPhase.java:217)
org.elasticsearch.action.search.InitialSearchPhase$2$$Lambda$2253/565438834.run(Unknown Source)
org.elasticsearch.action.search.InitialSearchPhase.maybeFork(InitialSearchPhase.java:171)
org.elasticsearch.action.search.InitialSearchPhase.access$000(InitialSearchPhase.java:49)
org.elasticsearch.action.search.InitialSearchPhase$2.onFailure(InitialSearchPhase.java:217)
org.elasticsearch.action.search.SearchExecutionStatsCollector.onFailure(SearchExecutionStatsCollector.java:73)
org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:51)
org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:527)
org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1098)
org.elasticsearch.transport.TcpTransport.lambda$handleException$33(TcpTransport.java:1478)
org.elasticsearch.transport.TcpTransport$$Lambda$2143/1290881415.run(Unknown Source)
org.elasticsearch.common.util.concurrent.EsExecutors$1.execute(EsExecutors.java:135)
org.elasticsearch.transport.TcpTransport.handleException(TcpTransport.java:1476)
org.elasticsearch.transport.TcpTransport.handlerResponseError(TcpTransport.java:1468)
org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1398)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:64)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:545)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:499)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
java.lang.Thread.run(Thread.java:748)
3/10 snapshots sharing following 27 elements
org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1398)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:64)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:545)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:499)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
java.lang.Thread.run(Thread.java:748)
::: {IjibcFs}{IjibcFs5ThuoUQx4tsXhxA}{lmc1tzD2Q_-TnviEUcLxoA}{x.x.x.x}{x.x.x.x:9300}{zone=eu-west-1c}
Hot threads at 2018-10-11T13:51:16.843Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
34.7% (173.3ms out of 500ms) cpu usage by thread 'elasticsearch[IjibcFs][search][T#2]'
2/10 snapshots sharing following 27 elements
org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
org.apache.lucene.index.LeafReader.docFreq(LeafReader.java:82)
org.apache.lucene.index.BaseCompositeReader.docFreq(BaseCompositeReader.java:126)
org.elasticsearch.common.lucene.search.XMoreLikeThis.createQueue(XMoreLikeThis.java:740)
org.elasticsearch.common.lucene.search.XMoreLikeThis.createQueue(XMoreLikeThis.java:715)
org.elasticsearch.common.lucene.search.XMoreLikeThis.like(XMoreLikeThis.java:629)
org.elasticsearch.common.lucene.search.MoreLikeThisQuery.createQuery(MoreLikeThisQuery.java:172)
org.elasticsearch.common.lucene.search.MoreLikeThisQuery.rewrite(MoreLikeThisQuery.java:156)
org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:243)
org.apache.lucene.search.IndexSearcher.rewrite(IndexSearcher.java:675)
org.elasticsearch.search.internal.ContextIndexSearcher.rewrite(ContextIndexSearcher.java:106)
org.elasticsearch.search.DefaultSearchContext.preProcess(DefaultSearchContext.java:257)
org.elasticsearch.search.query.QueryPhase.preProcess(QueryPhase.java:89)
org.elasticsearch.search.SearchService.createContext(SearchService.java:570)
org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:534)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:330)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
3/10 snapshots sharing following 28 elements
org.apache.lucene.util.fst.FST.findTargetArc(FST.java:1119)
org.apache.lucene.util.fst.FST.findTargetArc(FST.java:1044)
org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:483)
org.apache.lucene.index.LeafReader.docFreq(LeafReader.java:82)
org.apache.lucene.index.BaseCompositeReader.docFreq(BaseCompositeReader.java:126)
org.elasticsearch.common.lucene.search.XMoreLikeThis.createQueue(XMoreLikeThis.java:740)
org.elasticsearch.common.lucene.search.XMoreLikeThis.createQueue(XMoreLikeThis.java:715)
org.elasticsearch.common.lucene.search.XMoreLikeThis.like(XMoreLikeThis.java:629)
org.elasticsearch.common.lucene.search.MoreLikeThisQuery.createQuery(MoreLikeThisQuery.java:172)
org.elasticsearch.common.lucene.search.MoreLikeThisQuery.rewrite(MoreLikeThisQuery.java:156)
org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:243)
org.apache.lucene.search.IndexSearcher.rewrite(IndexSearcher.java:675)
org.elasticsearch.search.internal.ContextIndexSearcher.rewrite(ContextIndexSearcher.java:106)
org.elasticsearch.search.DefaultSearchContext.preProcess(DefaultSearchContext.java:257)
org.elasticsearch.search.query.QueryPhase.preProcess(QueryPhase.java:89)
org.elasticsearch.search.SearchService.createContext(SearchService.java:570)
org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:534)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:330)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
4/10 snapshots sharing following 27 elements
org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.decodeMetaData(SegmentTermsEnumFrame.java:428)
org.apache.lucene.codecs.blocktree.SegmentTermsEnum.docFreq(SegmentTermsEnum.java:980)
org.apache.lucene.index.LeafReader.docFreq(LeafReader.java:83)
org.apache.lucene.index.BaseCompositeReader.docFreq(BaseCompositeReader.java:126)
org.elasticsearch.common.lucene.search.XMoreLikeThis.createQueue(XMoreLikeThis.java:740)
org.elasticsearch.common.lucene.search.XMoreLikeThis.createQueue(XMoreLikeThis.java:715)
org.elasticsearch.common.lucene.search.XMoreLikeThis.like(XMoreLikeThis.java:629)
org.elasticsearch.common.lucene.search.MoreLikeThisQuery.createQuery(MoreLikeThisQuery.java:172)
org.elasticsearch.common.lucene.search.MoreLikeThisQuery.rewrite(MoreLikeThisQuery.java:156)
org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:243)
org.apache.lucene.search.IndexSearcher.rewrite(IndexSearcher.java:675)
org.elasticsearch.search.internal.ContextIndexSearcher.rewrite(ContextIndexSearcher.java:106)
org.elasticsearch.search.DefaultSearchContext.preProcess(DefaultSearchContext.java:257)
org.elasticsearch.search.query.QueryPhase.preProcess(QueryPhase.java:89)
org.elasticsearch.search.SearchService.createContext(SearchService.java:570)
org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:534)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:330)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
unique snapshot
org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.scanToTermLeaf(SegmentTermsEnumFrame.java:590)
org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.scanToTerm(SegmentTermsEnumFrame.java:481)
org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:509)
org.apache.lucene.index.LeafReader.docFreq(LeafReader.java:82)
org.apache.lucene.index.BaseCompositeReader.docFreq(BaseCompositeReader.java:126)
org.elasticsearch.common.lucene.search.XMoreLikeThis.createQueue(XMoreLikeThis.java:740)
org.elasticsearch.common.lucene.search.XMoreLikeThis.createQueue(XMoreLikeThis.java:715)
org.elasticsearch.common.lucene.search.XMoreLikeThis.like(XMoreLikeThis.java:629)
org.elasticsearch.common.lucene.search.MoreLikeThisQuery.createQuery(MoreLikeThisQuery.java:172)
org.elasticsearch.common.lucene.search.MoreLikeThisQuery.rewrite(MoreLikeThisQuery.java:156)
org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:243)
org.apache.lucene.search.IndexSearcher.rewrite(IndexSearcher.java:675)
org.elasticsearch.search.internal.ContextIndexSearcher.rewrite(ContextIndexSearcher.java:106)
org.elasticsearch.search.DefaultSearchContext.preProcess(DefaultSearchContext.java:257)
org.elasticsearch.search.query.QueryPhase.preProcess(QueryPhase.java:89)
org.elasticsearch.search.SearchService.createContext(SearchService.java:570)
org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:534)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:330)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
33.3% (166.3ms out of 500ms) cpu usage by thread 'elasticsearch[IjibcFs][search][T#7]'
5/10 snapshots sharing following 27 elements
org.apache.lucene.search.DisiPriorityQueue.downHeap(DisiPriorityQueue.java:158)
org.apache.lucene.search.DisiPriorityQueue.pop(DisiPriorityQueue.java:114)
org.apache.lucene.search.MinShouldMatchSumScorer.setDocAndFreq(MinShouldMatchSumScorer.java:266)
org.apache.lucene.search.MinShouldMatchSumScorer.access$100(MinShouldMatchSumScorer.java:49)
org.apache.lucene.search.MinShouldMatchSumScorer$2.advance(MinShouldMatchSumScorer.java:194)
org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:200)
org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:239)
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:184)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:265)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:307)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:340)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
3/10 snapshots sharing following 22 elements
org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
org.apache.lucene.search.TopScoreDocCollector$SimpleTopScoreDocCollector$1.collect(TopScoreDocCollector.java:64)
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:241)
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:184)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:265)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:307)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:340)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
2/10 snapshots sharing following 22 elements
org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:200)
org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:239)
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:184)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:265)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:307)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:340)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
32.4% (161.9ms out of 500ms) cpu usage by thread 'elasticsearch[IjibcFs][search][T#1]'
3/10 snapshots sharing following 27 elements
org.apache.lucene.search.DisiPriorityQueue.downHeap(DisiPriorityQueue.java:158)
org.apache.lucene.search.DisiPriorityQueue.pop(DisiPriorityQueue.java:114)
org.apache.lucene.search.MinShouldMatchSumScorer.setDocAndFreq(MinShouldMatchSumScorer.java:266)
org.apache.lucene.search.MinShouldMatchSumScorer.access$100(MinShouldMatchSumScorer.java:49)
org.apache.lucene.search.MinShouldMatchSumScorer$2.advance(MinShouldMatchSumScorer.java:194)
org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:200)
org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:239)
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:184)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:265)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:307)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:340)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
5/10 snapshots sharing following 21 elements
org.apache.lucene.search.TopScoreDocCollector$SimpleTopScoreDocCollector$1.collect(TopScoreDocCollector.java:64)
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:241)
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:184)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:265)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:307)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:340)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
2/10 snapshots sharing following 23 elements
org.apache.lucene.search.MinShouldMatchSumScorer$2.advance(MinShouldMatchSumScorer.java:177)
org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:200)
org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:239)
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:184)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:265)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:307)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:340)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
::: {SSt5weZ}{SSt5weZAQ3CSRKnM-xPZEQ}{AN3jTeX1SqWAGzRT8iiF2w}{x.x.x.x}{x.x.x.x:9300}{zone=eu-west-1c}
Hot threads at 2018-10-11T13:51:16.932Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
27.1% (135.2ms out of 500ms) cpu usage by thread 'elasticsearch[SSt5weZ][search][T#1]'
4/10 snapshots sharing following 22 elements
org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:200)
org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:239)
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:184)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:265)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:307)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:340)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
3/10 snapshots sharing following 25 elements
org.apache.lucene.search.similarities.BM25Similarity$BM25DocScorer.score(BM25Similarity.java:256)
org.apache.lucene.search.TermScorer.score(TermScorer.java:65)
org.apache.lucene.search.MinShouldMatchSumScorer.score(MinShouldMatchSumScorer.java:323)
org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
org.apache.lucene.search.TopScoreDocCollector$SimpleTopScoreDocCollector$1.collect(TopScoreDocCollector.java:64)
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:241)
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:184)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:265)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:307)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:340)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
2/10 snapshots sharing following 25 elements
org.apache.lucene.search.similarities.BM25Similarity$BM25DocScorer.score(BM25Similarity.java:257)
org.apache.lucene.search.TermScorer.score(TermScorer.java:65)
org.apache.lucene.search.MinShouldMatchSumScorer.score(MinShouldMatchSumScorer.java:323)
org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
org.apache.lucene.search.TopScoreDocCollector$SimpleTopScoreDocCollector$1.collect(TopScoreDocCollector.java:64)
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:241)
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:184)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:265)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:307)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:340)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
unique snapshot
org.apache.lucene.search.MinShouldMatchSumScorer.advanceTail(MinShouldMatchSumScorer.java:242)
org.apache.lucene.search.MinShouldMatchSumScorer.advanceTail(MinShouldMatchSumScorer.java:252)
org.apache.lucene.search.MinShouldMatchSumScorer.access$400(MinShouldMatchSumScorer.java:49)
org.apache.lucene.search.MinShouldMatchSumScorer$3.matches(MinShouldMatchSumScorer.java:212)
org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:240)
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:184)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:265)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:307)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:340)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
24.5% (122.6ms out of 500ms) cpu usage by thread 'elasticsearch[SSt5weZ][search][T#7]'
3/10 snapshots sharing following 22 elements
org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:200)
org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:239)
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:184)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:265)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:307)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:340)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
5/10 snapshots sharing following 27 elements
org.apache.lucene.codecs.lucene70.IndexedDISI.advanceExact(IndexedDISI.java:154)
org.apache.lucene.codecs.lucene70.Lucene70NormsProducer$SparseNormsIterator.advanceExact(Lucene70NormsProducer.java:163)
org.apache.lucene.search.similarities.BM25Similarity$BM25DocScorer.score(BM25Similarity.java:256)
org.apache.lucene.search.TermScorer.score(TermScorer.java:65)
org.apache.lucene.search.MinShouldMatchSumScorer.score(MinShouldMatchSumScorer.java:323)
org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
org.apache.lucene.search.TopScoreDocCollector$SimpleTopScoreDocCollector$1.collect(TopScoreDocCollector.java:64)
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:241)
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:184)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:265)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:307)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:340)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
2/10 snapshots sharing following 23 elements
org.apache.lucene.search.MinShouldMatchSumScorer.score(MinShouldMatchSumScorer.java:322)
org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
org.apache.lucene.search.TopScoreDocCollector$SimpleTopScoreDocCollector$1.collect(TopScoreDocCollector.java:64)
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:241)
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:184)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:265)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:307)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:340)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
22.9% (114.5ms out of 500ms) cpu usage by thread 'elasticsearch[SSt5weZ][search][T#5]'
3/10 snapshots sharing following 29 elements
org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.<init>(SegmentTermsEnumFrame.java:47)
org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getFrame(SegmentTermsEnum.java:215)
org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
org.apache.lucene.index.LeafReader.docFreq(LeafReader.java:82)
org.apache.lucene.index.BaseCompositeReader.docFreq(BaseCompositeReader.java:126)
org.elasticsearch.common.lucene.search.XMoreLikeThis.createQueue(XMoreLikeThis.java:740)
org.elasticsearch.common.lucene.search.XMoreLikeThis.createQueue(XMoreLikeThis.java:715)
org.elasticsearch.common.lucene.search.XMoreLikeThis.like(XMoreLikeThis.java:629)
org.elasticsearch.common.lucene.search.MoreLikeThisQuery.createQuery(MoreLikeThisQuery.java:172)
org.elasticsearch.common.lucene.search.MoreLikeThisQuery.rewrite(MoreLikeThisQuery.java:156)
org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:243)
org.apache.lucene.search.IndexSearcher.rewrite(IndexSearcher.java:675)
org.elasticsearch.search.internal.ContextIndexSearcher.rewrite(ContextIndexSearcher.java:106)
org.elasticsearch.search.DefaultSearchContext.preProcess(DefaultSearchContext.java:257)
org.elasticsearch.search.query.QueryPhase.preProcess(QueryPhase.java:89)
org.elasticsearch.search.SearchService.createContext(SearchService.java:570)
org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:534)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:330)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
3/10 snapshots sharing following 27 elements
org.apache.lucene.codecs.blocktree.SegmentTermsEnum.<init>(SegmentTermsEnum.java:77)
org.apache.lucene.codecs.blocktree.FieldReader.iterator(FieldReader.java:156)
org.apache.lucene.index.LeafReader.docFreq(LeafReader.java:81)
org.apache.lucene.index.BaseCompositeReader.docFreq(BaseCompositeReader.java:126)
org.elasticsearch.common.lucene.search.XMoreLikeThis.createQueue(XMoreLikeThis.java:740)
org.elasticsearch.common.lucene.search.XMoreLikeThis.createQueue(XMoreLikeThis.java:715)
org.elasticsearch.common.lucene.search.XMoreLikeThis.like(XMoreLikeThis.java:629)
org.elasticsearch.common.lucene.search.MoreLikeThisQuery.createQuery(MoreLikeThisQuery.java:172)
org.elasticsearch.common.lucene.search.MoreLikeThisQuery.rewrite(MoreLikeThisQuery.java:156)
org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:243)
org.apache.lucene.search.IndexSearcher.rewrite(IndexSearcher.java:675)
org.elasticsearch.search.internal.ContextIndexSearcher.rewrite(ContextIndexSearcher.java:106)
org.elasticsearch.search.DefaultSearchContext.preProcess(DefaultSearchContext.java:257)
org.elasticsearch.search.query.QueryPhase.preProcess(QueryPhase.java:89)
org.elasticsearch.search.SearchService.createContext(SearchService.java:570)
org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:534)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:330)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
2/10 snapshots sharing following 20 elements
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:241)
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:184)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:265)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:307)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:340)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
2/10 snapshots sharing following 27 elements
org.apache.lucene.search.MinShouldMatchSumScorer.upHeapCost(MinShouldMatchSumScorer.java:375)
org.apache.lucene.search.MinShouldMatchSumScorer.addTail(MinShouldMatchSumScorer.java:353)
org.apache.lucene.search.MinShouldMatchSumScorer.insertTailWithOverFlow(MinShouldMatchSumScorer.java:337)
org.apache.lucene.search.MinShouldMatchSumScorer.access$000(MinShouldMatchSumScorer.java:49)
org.apache.lucene.search.MinShouldMatchSumScorer$2.advance(MinShouldMatchSumScorer.java:175)
org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:200)
org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:239)
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:184)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:660)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:265)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:307)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:340)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:316)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:312)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1002)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment