Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Opensearch hot threads log
This file has been truncated, but you can view the full file.
::: {datanode109.xyz.com}{SgHVvZR8RI6-pH6u6hbCjA}{CS9DAWzeTrGI9d0-bqfD_w}{datanode109.xyz.com}{10.218.9.213:50141}{d}{shard_indexing_pressure_enabled=true}
Hot threads at 2022-08-31T09:35:52.708Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
61.4% (307.1ms out of 500ms) cpu usage by thread 'opensearch[datanode109.xyz.com][transport_worker][T#2]'
5/10 snapshots sharing following 103 elements
org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228)
org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201)
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x00000000000068cb.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x000000000000612c.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003297.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
2/10 snapshots sharing following 108 elements
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1510)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1433)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349)
org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242)
org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237)
org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201)
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x00000000000068cb.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x000000000000612c.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003297.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
2/10 snapshots sharing following 113 elements
org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$1$1358610711/0x00000000000068d7.apply(Unknown Source)
java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705)
org.opensearch.security.support.Base64Helper$DescriptorReplacer.replace(Base64Helper.java:160)
org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.writeClassDescriptor(Base64Helper.java:223)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeNonProxyDesc(ObjectOutputStream.java:1283)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeClassDesc(ObjectOutputStream.java:1232)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1428)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349)
org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242)
org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237)
org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201)
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x00000000000068cb.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x000000000000612c.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003297.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
unique snapshot
app//org.opensearch.search.SearchService.canMatch(SearchService.java:1275)
app//org.opensearch.search.SearchService.access$300(SearchService.java:154)
app//org.opensearch.search.SearchService$2.onResponse(SearchService.java:463)
app//org.opensearch.search.SearchService$2.onResponse(SearchService.java:454)
app//org.opensearch.search.SearchService.lambda$rewriteAndFetchShardRequest$8(SearchService.java:1336)
app//org.opensearch.search.SearchService$$Lambda$lambda$rewriteAndFetchShardRequest$8$2844487977/0x00000000000065fe.accept(Unknown Source)
app//org.opensearch.index.shard.IndexShard.awaitShardSearchActive(IndexShard.java:3825)
app//org.opensearch.search.SearchService.lambda$rewriteAndFetchShardRequest$9(SearchService.java:1336)
app//org.opensearch.search.SearchService$$Lambda$lambda$rewriteAndFetchShardRequest$9$1490963251/0x000000000000611b.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.search.SearchService.rewriteAndFetchShardRequest(SearchService.java:1342)
app//org.opensearch.search.SearchService.executeQueryPhase(SearchService.java:454)
app//org.opensearch.action.search.SearchTransportService.lambda$registerRequestHandler$6(SearchTransportService.java:452)
app//org.opensearch.action.search.SearchTransportService$$Lambda$lambda$registerRequestHandler$6$421376029/0x0000000000005745.messageReceived(Unknown Source)
org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:139)
org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193)
org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:184)
org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97)
org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647)
app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91)
app//org.opensearch.transport.TransportService.sendLocalRequest(TransportService.java:938)
app//org.opensearch.transport.TransportService.access$100(TransportService.java:92)
app//org.opensearch.transport.TransportService$3.sendRequest(TransportService.java:151)
app//org.opensearch.transport.TransportService.sendRequestInternal(TransportService.java:876)
app//org.opensearch.transport.TransportService$$Lambda$sendRequestInternal$2649957159/0x00000000000052c2.sendRequest(Unknown Source)
org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:207)
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x00000000000068cb.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x000000000000612c.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003297.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
::: {datanode102.xyz.com}{l7Euuj60RbqQTC1aavG6Ig}{2RtDXmisSZWRlss3EKafCQ}{datanode102.xyz.com}{10.218.9.206:50141}{dm}{shard_indexing_pressure_enabled=true}
Hot threads at 2022-08-31T09:35:52.920Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
21.0% (104.7ms out of 500ms) cpu usage by thread 'opensearch[datanode102.xyz.com][transport_worker][T#2]'
9/10 snapshots sharing following 3 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
::: {datanode101.xyz.com}{9IHR5tYxRNerG7R5OtxsvQ}{EGoQdq1oREmP_7NLJeaHiQ}{datanode101.xyz.com}{10.218.9.205:50141}{dm}{shard_indexing_pressure_enabled=true}
Hot threads at 2022-08-31T09:35:53.085Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
89.6% (447.7ms out of 500ms) cpu usage by thread 'opensearch[datanode101.xyz.com][transport_worker][T#10]'
5/10 snapshots sharing following 106 elements
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349)
org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242)
org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237)
org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201)
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000612f.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000061a7.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005c43.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
2/10 snapshots sharing following 105 elements
org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199)
org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241)
org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228)
org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201)
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000612f.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000061a7.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005c43.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
3/10 snapshots sharing following 101 elements
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000612f.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000061a7.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005c43.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
7.2% (36ms out of 500ms) cpu usage by thread 'opensearch[datanode101.xyz.com][transport_worker][T#7]'
6/10 snapshots sharing following 3 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
::: {datanode103.xyz.com}{Lvc0FiOTRlShBEDWa4XtQQ}{-mLQ-ZHbSgOyGS6ptZPLhQ}{datanode103.xyz.com}{10.218.9.207:50141}{dm}{shard_indexing_pressure_enabled=true}
Hot threads at 2022-08-31T09:35:52.437Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
57.7% (288.3ms out of 500ms) cpu usage by thread 'opensearch[datanode103.xyz.com][transport_worker][T#6]'
5/10 snapshots sharing following 113 elements
org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$1$1358610711/0x0000000000006832.apply(Unknown Source)
java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705)
org.opensearch.security.support.Base64Helper$DescriptorReplacer.replace(Base64Helper.java:160)
org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.writeClassDescriptor(Base64Helper.java:223)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeNonProxyDesc(ObjectOutputStream.java:1283)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeClassDesc(ObjectOutputStream.java:1232)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1428)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349)
org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242)
org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237)
org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201)
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006811.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067b4.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000006663.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
5/10 snapshots sharing following 101 elements
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006811.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067b4.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000006663.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
22.2% (110.8ms out of 500ms) cpu usage by thread 'opensearch[datanode103.xyz.com][transport_worker][T#7]'
5/10 snapshots sharing following 3 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
::: {datanode106.xyz.com}{XPP-bmQATm2ygjbpSUWuHQ}{kim-kJzoQLeWtV3htztH8g}{datanode106.xyz.com}{10.218.9.210:50141}{d}{shard_indexing_pressure_enabled=true}
Hot threads at 2022-08-31T09:35:52.735Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
50.1% (250.3ms out of 500ms) cpu usage by thread 'opensearch[datanode106.xyz.com][transport_worker][T#4]'
4/10 snapshots sharing following 113 elements
org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$1$1358610711/0x00000000000065cf.apply(Unknown Source)
java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705)
org.opensearch.security.support.Base64Helper$DescriptorReplacer.replace(Base64Helper.java:160)
org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.writeClassDescriptor(Base64Helper.java:223)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeNonProxyDesc(ObjectOutputStream.java:1283)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeClassDesc(ObjectOutputStream.java:1232)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1428)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349)
org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242)
org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237)
org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201)
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000657f.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x0000000000006126.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003249.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
3/10 snapshots sharing following 105 elements
org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199)
org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241)
org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228)
org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201)
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000657f.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x0000000000006126.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003249.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
3/10 snapshots sharing following 101 elements
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000657f.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x0000000000006126.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003249.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
::: {datanode105.xyz.com}{naQqgBCNREGHSOQlJ4-XaQ}{p9c9s3RMTkWPX_lY4fU7VA}{datanode105.xyz.com}{10.218.9.209:50141}{d}{shard_indexing_pressure_enabled=true}
Hot threads at 2022-08-31T09:35:52.791Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
53.9% (269.2ms out of 500ms) cpu usage by thread 'opensearch[datanode105.xyz.com][transport_worker][T#7]'
3/10 snapshots sharing following 105 elements
org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199)
org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241)
org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228)
org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201)
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000684e.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c4.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
2/10 snapshots sharing following 123 elements
java.base@11.0.14.1.101/java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:2133)
java.base@11.0.14.1.101/java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:2000)
java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1795)
java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:608)
java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:566)
org.apache.commons.lang3.SerializationUtils.clone(SerializationUtils.java:95)
org.opensearch.security.support.Base64Helper$DescriptorReplacer.lambda$replace$0(Base64Helper.java:164)
org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$0$883570508/0x000000000000685a.run(Unknown Source)
java.base@11.0.14.1.101/java.security.AccessController.doPrivileged(Native Method)
org.opensearch.security.support.Base64Helper$DescriptorReplacer.lambda$replace$1(Base64Helper.java:163)
org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$1$1358610711/0x0000000000006859.apply(Unknown Source)
java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705)
org.opensearch.security.support.Base64Helper$DescriptorReplacer.replace(Base64Helper.java:160)
org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.writeClassDescriptor(Base64Helper.java:223)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeNonProxyDesc(ObjectOutputStream.java:1283)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeClassDesc(ObjectOutputStream.java:1232)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1428)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349)
org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242)
org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237)
org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201)
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000684e.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c4.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
4/10 snapshots sharing following 103 elements
org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237)
org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201)
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000684e.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c4.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
unique snapshot
java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.putVal(ConcurrentHashMap.java:1016)
java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.put(ConcurrentHashMap.java:1006)
app//org.opensearch.common.util.concurrent.ConcurrentHashMapLong.put(ConcurrentHashMapLong.java:60)
app//org.opensearch.transport.Transport$ResponseHandlers.add(Transport.java:217)
app//org.opensearch.transport.TransportService.sendRequestInternal(TransportService.java:856)
app//org.opensearch.transport.TransportService$$Lambda$sendRequestInternal$2649957159/0x00000000000050f3.sendRequest(Unknown Source)
org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:207)
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000684e.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c4.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
22.5% (112.6ms out of 500ms) cpu usage by thread 'opensearch[datanode105.xyz.com][transport_worker][T#10]'
5/10 snapshots sharing following 3 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
18.8% (94.1ms out of 500ms) cpu usage by thread 'opensearch[datanode105.xyz.com][transport_worker][T#9]'
10/10 snapshots sharing following 3 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
::: {datanode108.xyz.com}{-dUr4PyJQIixE17KOTDoPw}{GsiUAVUiRwWt5ieSs3FTPQ}{datanode108.xyz.com}{10.218.9.212:50141}{d}{shard_indexing_pressure_enabled=true}
Hot threads at 2022-08-31T09:35:53.175Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
65.8% (329.1ms out of 500ms) cpu usage by thread 'opensearch[datanode108.xyz.com][transport_worker][T#5]'
2/10 snapshots sharing following 125 elements
java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1817)
java.base@11.0.14.1.101/java.io.ObjectInputStream$GetFieldImpl.readFields(ObjectInputStream.java:2812)
java.base@11.0.14.1.101/java.io.ObjectInputStream.readFields(ObjectInputStream.java:788)
java.base@11.0.14.1.101/java.net.InetSocketAddress.readObject(InetSocketAddress.java:282)
java.base@11.0.14.1.101/jdk.internal.reflect.GeneratedMethodAccessor46.invoke(Unknown Source)
java.base@11.0.14.1.101/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base@11.0.14.1.101/java.lang.reflect.Method.invoke(Method.java:566)
java.base@11.0.14.1.101/java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1175)
java.base@11.0.14.1.101/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2487)
java.base@11.0.14.1.101/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2358)
java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1817)
java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:608)
java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:566)
org.opensearch.security.support.Base64Helper.deserializeObject(Base64Helper.java:257)
org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:170)
org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97)
org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647)
app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91)
app//org.opensearch.transport.TransportService.sendLocalRequest(TransportService.java:938)
app//org.opensearch.transport.TransportService.access$100(TransportService.java:92)
app//org.opensearch.transport.TransportService$3.sendRequest(TransportService.java:151)
app//org.opensearch.transport.TransportService.sendRequestInternal(TransportService.java:876)
app//org.opensearch.transport.TransportService$$Lambda$sendRequestInternal$2649957159/0x00000000000052c3.sendRequest(Unknown Source)
org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:207)
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006905.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000068b6.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x000000000000613e.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
4/10 snapshots sharing following 106 elements
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179)
java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349)
org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242)
org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237)
org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201)
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006905.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000068b6.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x000000000000613e.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
2/10 snapshots sharing following 102 elements
org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201)
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006905.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000068b6.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x000000000000613e.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
2/10 snapshots sharing following 96 elements
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006905.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000068b6.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x000000000000613e.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
16.0% (79.9ms out of 500ms) cpu usage by thread 'opensearch[datanode108.xyz.com][transport_worker][T#6]'
5/10 snapshots sharing following 3 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
::: {datanode107.xyz.com}{VIusRcUdRoW7fDPV4ybdig}{7q0H0yXvQo2NYLe9eihLCw}{datanode107.xyz.com}{10.218.9.211:50141}{d}{shard_indexing_pressure_enabled=true}
Hot threads at 2022-08-31T09:35:52.803Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
52.1% (260.5ms out of 500ms) cpu usage by thread 'opensearch[datanode107.xyz.com][transport_worker][T#1]'
4/10 snapshots sharing following 103 elements
org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228)
org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201)
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000691e.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249)
app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415)
app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215)
app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994)
app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757)
app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398)
app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000068ab.accept(Unknown Source)
app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136)
app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101)
app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277)
app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194)
org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319)
org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154)
app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169)
app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97)
app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108)
app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104)
app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433)
app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137)
app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003378.accept(Unknown Source)
app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128)
org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306)
app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392)
app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235)
org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63)
app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361)
app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440)
app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55)
org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887)
6/10 snapshots sharing following 101 elements
org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661)
app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838)
app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826)
app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196)
app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124)
app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278)
app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000691e.run(Unknown Source)
app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773)
app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310)
app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearc