Created
August 31, 2022 21:47
-
-
Save TysonHeart/04bcb716223d74125d04809d44b41ec5 to your computer and use it in GitHub Desktop.
Opensearch hot threads log
This file has been truncated, but you can view the full file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ::: {datanode109.xyz.com}{SgHVvZR8RI6-pH6u6hbCjA}{CS9DAWzeTrGI9d0-bqfD_w}{datanode109.xyz.com}{10.218.9.213:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:52.708Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 61.4% (307.1ms out of 500ms) cpu usage by thread 'opensearch[datanode109.xyz.com][transport_worker][T#2]' | |
| 5/10 snapshots sharing following 103 elements | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x00000000000068cb.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x000000000000612c.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003297.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 108 elements | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1510) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1433) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x00000000000068cb.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x000000000000612c.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003297.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 113 elements | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$1$1358610711/0x00000000000068d7.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.replace(Base64Helper.java:160) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.writeClassDescriptor(Base64Helper.java:223) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeNonProxyDesc(ObjectOutputStream.java:1283) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeClassDesc(ObjectOutputStream.java:1232) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1428) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x00000000000068cb.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x000000000000612c.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003297.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| unique snapshot | |
| app//org.opensearch.search.SearchService.canMatch(SearchService.java:1275) | |
| app//org.opensearch.search.SearchService.access$300(SearchService.java:154) | |
| app//org.opensearch.search.SearchService$2.onResponse(SearchService.java:463) | |
| app//org.opensearch.search.SearchService$2.onResponse(SearchService.java:454) | |
| app//org.opensearch.search.SearchService.lambda$rewriteAndFetchShardRequest$8(SearchService.java:1336) | |
| app//org.opensearch.search.SearchService$$Lambda$lambda$rewriteAndFetchShardRequest$8$2844487977/0x00000000000065fe.accept(Unknown Source) | |
| app//org.opensearch.index.shard.IndexShard.awaitShardSearchActive(IndexShard.java:3825) | |
| app//org.opensearch.search.SearchService.lambda$rewriteAndFetchShardRequest$9(SearchService.java:1336) | |
| app//org.opensearch.search.SearchService$$Lambda$lambda$rewriteAndFetchShardRequest$9$1490963251/0x000000000000611b.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.search.SearchService.rewriteAndFetchShardRequest(SearchService.java:1342) | |
| app//org.opensearch.search.SearchService.executeQueryPhase(SearchService.java:454) | |
| app//org.opensearch.action.search.SearchTransportService.lambda$registerRequestHandler$6(SearchTransportService.java:452) | |
| app//org.opensearch.action.search.SearchTransportService$$Lambda$lambda$registerRequestHandler$6$421376029/0x0000000000005745.messageReceived(Unknown Source) | |
| org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:139) | |
| org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) | |
| org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:184) | |
| org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.TransportService.sendLocalRequest(TransportService.java:938) | |
| app//org.opensearch.transport.TransportService.access$100(TransportService.java:92) | |
| app//org.opensearch.transport.TransportService$3.sendRequest(TransportService.java:151) | |
| app//org.opensearch.transport.TransportService.sendRequestInternal(TransportService.java:876) | |
| app//org.opensearch.transport.TransportService$$Lambda$sendRequestInternal$2649957159/0x00000000000052c2.sendRequest(Unknown Source) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:207) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x00000000000068cb.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x000000000000612c.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003297.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode102.xyz.com}{l7Euuj60RbqQTC1aavG6Ig}{2RtDXmisSZWRlss3EKafCQ}{datanode102.xyz.com}{10.218.9.206:50141}{dm}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:52.920Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 21.0% (104.7ms out of 500ms) cpu usage by thread 'opensearch[datanode102.xyz.com][transport_worker][T#2]' | |
| 9/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode101.xyz.com}{9IHR5tYxRNerG7R5OtxsvQ}{EGoQdq1oREmP_7NLJeaHiQ}{datanode101.xyz.com}{10.218.9.205:50141}{dm}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:53.085Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 89.6% (447.7ms out of 500ms) cpu usage by thread 'opensearch[datanode101.xyz.com][transport_worker][T#10]' | |
| 5/10 snapshots sharing following 106 elements | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000612f.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000061a7.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005c43.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 105 elements | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000612f.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000061a7.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005c43.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 3/10 snapshots sharing following 101 elements | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000612f.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000061a7.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005c43.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 7.2% (36ms out of 500ms) cpu usage by thread 'opensearch[datanode101.xyz.com][transport_worker][T#7]' | |
| 6/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode103.xyz.com}{Lvc0FiOTRlShBEDWa4XtQQ}{-mLQ-ZHbSgOyGS6ptZPLhQ}{datanode103.xyz.com}{10.218.9.207:50141}{dm}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:52.437Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 57.7% (288.3ms out of 500ms) cpu usage by thread 'opensearch[datanode103.xyz.com][transport_worker][T#6]' | |
| 5/10 snapshots sharing following 113 elements | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$1$1358610711/0x0000000000006832.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.replace(Base64Helper.java:160) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.writeClassDescriptor(Base64Helper.java:223) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeNonProxyDesc(ObjectOutputStream.java:1283) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeClassDesc(ObjectOutputStream.java:1232) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1428) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006811.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067b4.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000006663.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 5/10 snapshots sharing following 101 elements | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006811.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067b4.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000006663.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 22.2% (110.8ms out of 500ms) cpu usage by thread 'opensearch[datanode103.xyz.com][transport_worker][T#7]' | |
| 5/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode106.xyz.com}{XPP-bmQATm2ygjbpSUWuHQ}{kim-kJzoQLeWtV3htztH8g}{datanode106.xyz.com}{10.218.9.210:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:52.735Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 50.1% (250.3ms out of 500ms) cpu usage by thread 'opensearch[datanode106.xyz.com][transport_worker][T#4]' | |
| 4/10 snapshots sharing following 113 elements | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$1$1358610711/0x00000000000065cf.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.replace(Base64Helper.java:160) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.writeClassDescriptor(Base64Helper.java:223) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeNonProxyDesc(ObjectOutputStream.java:1283) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeClassDesc(ObjectOutputStream.java:1232) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1428) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000657f.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x0000000000006126.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003249.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 3/10 snapshots sharing following 105 elements | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000657f.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x0000000000006126.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003249.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 3/10 snapshots sharing following 101 elements | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000657f.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x0000000000006126.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003249.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode105.xyz.com}{naQqgBCNREGHSOQlJ4-XaQ}{p9c9s3RMTkWPX_lY4fU7VA}{datanode105.xyz.com}{10.218.9.209:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:52.791Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 53.9% (269.2ms out of 500ms) cpu usage by thread 'opensearch[datanode105.xyz.com][transport_worker][T#7]' | |
| 3/10 snapshots sharing following 105 elements | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000684e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c4.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 123 elements | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:2133) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:2000) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1795) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:608) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:566) | |
| org.apache.commons.lang3.SerializationUtils.clone(SerializationUtils.java:95) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.lambda$replace$0(Base64Helper.java:164) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$0$883570508/0x000000000000685a.run(Unknown Source) | |
| java.base@11.0.14.1.101/java.security.AccessController.doPrivileged(Native Method) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.lambda$replace$1(Base64Helper.java:163) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$1$1358610711/0x0000000000006859.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.replace(Base64Helper.java:160) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.writeClassDescriptor(Base64Helper.java:223) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeNonProxyDesc(ObjectOutputStream.java:1283) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeClassDesc(ObjectOutputStream.java:1232) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1428) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000684e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c4.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 4/10 snapshots sharing following 103 elements | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000684e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c4.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| unique snapshot | |
| java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.putVal(ConcurrentHashMap.java:1016) | |
| java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.put(ConcurrentHashMap.java:1006) | |
| app//org.opensearch.common.util.concurrent.ConcurrentHashMapLong.put(ConcurrentHashMapLong.java:60) | |
| app//org.opensearch.transport.Transport$ResponseHandlers.add(Transport.java:217) | |
| app//org.opensearch.transport.TransportService.sendRequestInternal(TransportService.java:856) | |
| app//org.opensearch.transport.TransportService$$Lambda$sendRequestInternal$2649957159/0x00000000000050f3.sendRequest(Unknown Source) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:207) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000684e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c4.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 22.5% (112.6ms out of 500ms) cpu usage by thread 'opensearch[datanode105.xyz.com][transport_worker][T#10]' | |
| 5/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 18.8% (94.1ms out of 500ms) cpu usage by thread 'opensearch[datanode105.xyz.com][transport_worker][T#9]' | |
| 10/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode108.xyz.com}{-dUr4PyJQIixE17KOTDoPw}{GsiUAVUiRwWt5ieSs3FTPQ}{datanode108.xyz.com}{10.218.9.212:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:53.175Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 65.8% (329.1ms out of 500ms) cpu usage by thread 'opensearch[datanode108.xyz.com][transport_worker][T#5]' | |
| 2/10 snapshots sharing following 125 elements | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1817) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream$GetFieldImpl.readFields(ObjectInputStream.java:2812) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readFields(ObjectInputStream.java:788) | |
| java.base@11.0.14.1.101/java.net.InetSocketAddress.readObject(InetSocketAddress.java:282) | |
| java.base@11.0.14.1.101/jdk.internal.reflect.GeneratedMethodAccessor46.invoke(Unknown Source) | |
| java.base@11.0.14.1.101/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| java.base@11.0.14.1.101/java.lang.reflect.Method.invoke(Method.java:566) | |
| java.base@11.0.14.1.101/java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1175) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2487) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2358) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1817) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:608) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:566) | |
| org.opensearch.security.support.Base64Helper.deserializeObject(Base64Helper.java:257) | |
| org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:170) | |
| org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.TransportService.sendLocalRequest(TransportService.java:938) | |
| app//org.opensearch.transport.TransportService.access$100(TransportService.java:92) | |
| app//org.opensearch.transport.TransportService$3.sendRequest(TransportService.java:151) | |
| app//org.opensearch.transport.TransportService.sendRequestInternal(TransportService.java:876) | |
| app//org.opensearch.transport.TransportService$$Lambda$sendRequestInternal$2649957159/0x00000000000052c3.sendRequest(Unknown Source) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:207) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006905.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000068b6.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x000000000000613e.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 4/10 snapshots sharing following 106 elements | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006905.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000068b6.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x000000000000613e.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 102 elements | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006905.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000068b6.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x000000000000613e.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 96 elements | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006905.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000068b6.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x000000000000613e.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 16.0% (79.9ms out of 500ms) cpu usage by thread 'opensearch[datanode108.xyz.com][transport_worker][T#6]' | |
| 5/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode107.xyz.com}{VIusRcUdRoW7fDPV4ybdig}{7q0H0yXvQo2NYLe9eihLCw}{datanode107.xyz.com}{10.218.9.211:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:52.803Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 52.1% (260.5ms out of 500ms) cpu usage by thread 'opensearch[datanode107.xyz.com][transport_worker][T#1]' | |
| 4/10 snapshots sharing following 103 elements | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000691e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000068ab.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003378.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 6/10 snapshots sharing following 101 elements | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000691e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000068ab.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003378.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 27.7% (138.7ms out of 500ms) cpu usage by thread 'opensearch[datanode107.xyz.com][transport_worker][T#5]' | |
| 9/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode104.xyz.com}{7wGFZrOLSiCpjhSzFsq9bQ}{iX2EGtP7SnyWXnKvhJCFcQ}{datanode104.xyz.com}{10.218.9.208:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:53.160Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 60.4% (302ms out of 500ms) cpu usage by thread 'opensearch[datanode104.xyz.com][transport_worker][T#9]' | |
| 3/10 snapshots sharing following 124 elements | |
| org.apache.commons.lang3.SerializationUtils$ClassLoaderAwareObjectInputStream.resolveClass(SerializationUtils.java:325) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:2133) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:2000) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1795) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:608) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:566) | |
| org.apache.commons.lang3.SerializationUtils.clone(SerializationUtils.java:95) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.lambda$replace$0(Base64Helper.java:164) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$0$883570508/0x0000000000006847.run(Unknown Source) | |
| java.base@11.0.14.1.101/java.security.AccessController.doPrivileged(Native Method) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.lambda$replace$1(Base64Helper.java:163) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$1$1358610711/0x0000000000006834.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.replace(Base64Helper.java:160) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.writeClassDescriptor(Base64Helper.java:223) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeNonProxyDesc(ObjectOutputStream.java:1283) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeClassDesc(ObjectOutputStream.java:1232) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1428) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006829.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c9.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000006059.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 7/10 snapshots sharing following 101 elements | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006829.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c9.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000006059.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 26.0% (129.8ms out of 500ms) cpu usage by thread 'opensearch[datanode104.xyz.com][transport_worker][T#6]' | |
| 9/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode110.xyz.com}{RDbmkqioTOqmgCCcOFtaCQ}{2e8szEuPRyStg-RmZN9tjw}{datanode110.xyz.com}{10.218.9.214:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:52.928Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 36.9% (184.5ms out of 500ms) cpu usage by thread 'opensearch[datanode110.xyz.com][transport_worker][T#10]' | |
| 8/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 28.3% (141.5ms out of 500ms) cpu usage by thread 'opensearch[datanode110.xyz.com][transport_worker][T#8]' | |
| 2/10 snapshots sharing following 46 elements | |
| org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) | |
| org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:336) | |
| org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:153) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.InboundHandler.handleRequest(InboundHandler.java:268) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:126) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d50.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 4/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 43 elements | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.InboundHandler.handleRequest(InboundHandler.java:268) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:126) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d50.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 27.5% (137.7ms out of 500ms) cpu usage by thread 'opensearch[datanode110.xyz.com][transport_worker][T#4]' | |
| 3/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode109.xyz.com}{SgHVvZR8RI6-pH6u6hbCjA}{CS9DAWzeTrGI9d0-bqfD_w}{datanode109.xyz.com}{10.218.9.213:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:55.523Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 67.5% (337.7ms out of 500ms) cpu usage by thread 'opensearch[datanode109.xyz.com][transport_worker][T#6]' | |
| 2/10 snapshots sharing following 123 elements | |
| java.base@11.0.14.1.101/java.lang.Throwable.fillInStackTrace(Native Method) | |
| java.base@11.0.14.1.101/java.lang.Throwable.fillInStackTrace(Throwable.java:787) | |
| java.base@11.0.14.1.101/java.lang.Throwable.<init>(Throwable.java:270) | |
| java.base@11.0.14.1.101/java.lang.Exception.<init>(Exception.java:66) | |
| java.base@11.0.14.1.101/java.io.IOException.<init>(IOException.java:58) | |
| java.base@11.0.14.1.101/java.io.PrintStream.ensureOpen(PrintStream.java:404) | |
| java.base@11.0.14.1.101/java.io.PrintStream.write(PrintStream.java:603) | |
| java.base@11.0.14.1.101/java.io.PrintStream.print(PrintStream.java:745) | |
| java.base@11.0.14.1.101/java.io.PrintStream.println(PrintStream.java:882) | |
| java.base@11.0.14.1.101/sun.security.util.Debug.println(Debug.java:184) | |
| java.base@11.0.14.1.101/java.security.AccessControlContext.checkPermission(AccessControlContext.java:478) | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:897) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.enableReplaceObject(ObjectOutputStream.java:623) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.lambda$new$1(Base64Helper.java:217) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream$$Lambda$lambda$new$1$486334844/0x00000000000060a2.run(Unknown Source) | |
| java.base@11.0.14.1.101/java.security.AccessController.doPrivileged(Native Method) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.<init>(Base64Helper.java:216) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x00000000000068cb.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x000000000000612c.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003297.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 105 elements | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x00000000000068cb.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x000000000000612c.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003297.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 113 elements | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$1$1358610711/0x00000000000068d7.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.replace(Base64Helper.java:160) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.writeClassDescriptor(Base64Helper.java:223) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeNonProxyDesc(ObjectOutputStream.java:1283) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeClassDesc(ObjectOutputStream.java:1232) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1428) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x00000000000068cb.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x000000000000612c.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003297.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 110 elements | |
| org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.TransportService.sendLocalRequest(TransportService.java:938) | |
| app//org.opensearch.transport.TransportService.access$100(TransportService.java:92) | |
| app//org.opensearch.transport.TransportService$3.sendRequest(TransportService.java:151) | |
| app//org.opensearch.transport.TransportService.sendRequestInternal(TransportService.java:876) | |
| app//org.opensearch.transport.TransportService$$Lambda$sendRequestInternal$2649957159/0x00000000000052c2.sendRequest(Unknown Source) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:207) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x00000000000068cb.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x000000000000612c.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003297.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 105 elements | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x00000000000068cb.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x000000000000612c.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003297.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 7.0% (35ms out of 500ms) cpu usage by thread 'opensearch[datanode109.xyz.com][transport_worker][T#2]' | |
| 10/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode102.xyz.com}{l7Euuj60RbqQTC1aavG6Ig}{2RtDXmisSZWRlss3EKafCQ}{datanode102.xyz.com}{10.218.9.206:50141}{dm}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:55.735Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 18.3% (91.7ms out of 500ms) cpu usage by thread 'opensearch[datanode102.xyz.com][transport_worker][T#2]' | |
| 4/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode101.xyz.com}{9IHR5tYxRNerG7R5OtxsvQ}{EGoQdq1oREmP_7NLJeaHiQ}{datanode101.xyz.com}{10.218.9.205:50141}{dm}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:56.371Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 21.4% (106.8ms out of 500ms) cpu usage by thread 'opensearch[datanode101.xyz.com][transport_worker][T#7]' | |
| 4/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 19.2% (96.1ms out of 500ms) cpu usage by thread 'opensearch[datanode101.xyz.com][transport_worker][T#5]' | |
| 7/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 16.7% (83.7ms out of 500ms) cpu usage by thread 'opensearch[datanode101.xyz.com][transport_worker][T#3]' | |
| 2/10 snapshots sharing following 89 elements | |
| java.base@11.0.14.1.101/java.util.Collections$UnmodifiableMap$UnmodifiableEntrySet.lambda$entryConsumer$0(Collections.java:1576) | |
| java.base@11.0.14.1.101/java.util.Collections$UnmodifiableMap$UnmodifiableEntrySet$$Lambda$lambda$entryConsumer$0$1860662117/0x0000000000004927.accept(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.TreeMap$EntrySpliterator.forEachRemaining(TreeMap.java:2962) | |
| java.base@11.0.14.1.101/java.util.Collections$UnmodifiableMap$UnmodifiableEntrySet$UnmodifiableEntrySetSpliterator.forEachRemaining(Collections.java:1601) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) | |
| java.base@11.0.14.1.101/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:913) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) | |
| java.base@11.0.14.1.101/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578) | |
| org.opensearch.security.resolver.IndexResolverReplacer$ResolvedIndicesProvider.resolveIndexPatterns(IndexResolverReplacer.java:257) | |
| org.opensearch.security.resolver.IndexResolverReplacer$ResolvedIndicesProvider.provide(IndexResolverReplacer.java:305) | |
| org.opensearch.security.resolver.IndexResolverReplacer.getOrReplaceAllIndices(IndexResolverReplacer.java:642) | |
| org.opensearch.security.resolver.IndexResolverReplacer.resolveRequest(IndexResolverReplacer.java:350) | |
| org.opensearch.security.privileges.PrivilegesEvaluator.evaluate(PrivilegesEvaluator.java:263) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:305) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005c43.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 89 elements | |
| org.opensearch.security.securityconf.ConfigModelV7$IndexPattern.access$400(ConfigModelV7.java:643) | |
| org.opensearch.security.securityconf.ConfigModelV7.lambda$impliesTypePerm$4(ConfigModelV7.java:965) | |
| org.opensearch.security.securityconf.ConfigModelV7$$Lambda$lambda$impliesTypePerm$4$590894108/0x0000000000005c55.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195) | |
| java.base@11.0.14.1.101/java.util.HashMap$KeySpliterator.forEachRemaining(HashMap.java:1621) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:550) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.evaluateToArrayNode(AbstractPipeline.java:260) | |
| java.base@11.0.14.1.101/java.util.stream.ReferencePipeline.toArray(ReferencePipeline.java:517) | |
| org.opensearch.security.securityconf.ConfigModelV7.impliesTypePerm(ConfigModelV7.java:966) | |
| org.opensearch.security.securityconf.ConfigModelV7.access$600(ConfigModelV7.java:68) | |
| org.opensearch.security.securityconf.ConfigModelV7$SecurityRoles.impliesTypePermGlobal(ConfigModelV7.java:487) | |
| org.opensearch.security.privileges.PrivilegesEvaluator.evaluate(PrivilegesEvaluator.java:460) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:305) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005c43.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode103.xyz.com}{Lvc0FiOTRlShBEDWa4XtQQ}{-mLQ-ZHbSgOyGS6ptZPLhQ}{datanode103.xyz.com}{10.218.9.207:50141}{dm}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:55.245Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 7.8% (39.1ms out of 500ms) cpu usage by thread 'opensearch[datanode103.xyz.com][transport_worker][T#10]' | |
| 2/10 snapshots sharing following 43 elements | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.InboundHandler.handleRequest(InboundHandler.java:268) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:126) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005b7a.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 6/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode106.xyz.com}{XPP-bmQATm2ygjbpSUWuHQ}{kim-kJzoQLeWtV3htztH8g}{datanode106.xyz.com}{10.218.9.210:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:55.547Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 11.0% (55.1ms out of 500ms) cpu usage by thread 'opensearch[datanode106.xyz.com][transport_worker][T#5]' | |
| 10/10 snapshots sharing following 75 elements | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:305) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003249.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode105.xyz.com}{naQqgBCNREGHSOQlJ4-XaQ}{p9c9s3RMTkWPX_lY4fU7VA}{datanode105.xyz.com}{10.218.9.209:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:55.599Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 67.7% (338.3ms out of 500ms) cpu usage by thread 'opensearch[datanode105.xyz.com][transport_worker][T#8]' | |
| 2/10 snapshots sharing following 118 elements | |
| java.base@11.0.14.1.101/java.lang.Throwable.fillInStackTrace(Native Method) | |
| java.base@11.0.14.1.101/java.lang.Throwable.fillInStackTrace(Throwable.java:787) | |
| java.base@11.0.14.1.101/java.lang.Throwable.<init>(Throwable.java:270) | |
| java.base@11.0.14.1.101/java.lang.Exception.<init>(Exception.java:66) | |
| java.base@11.0.14.1.101/java.io.IOException.<init>(IOException.java:58) | |
| java.base@11.0.14.1.101/java.io.PrintStream.ensureOpen(PrintStream.java:404) | |
| java.base@11.0.14.1.101/java.io.PrintStream.newLine(PrintStream.java:622) | |
| java.base@11.0.14.1.101/java.io.PrintStream.println(PrintStream.java:883) | |
| java.base@11.0.14.1.101/sun.security.util.Debug.println(Debug.java:184) | |
| java.base@11.0.14.1.101/java.security.AccessControlContext.checkPermission(AccessControlContext.java:478) | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:897) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.<init>(Base64Helper.java:213) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000684e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c4.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 7/10 snapshots sharing following 102 elements | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000684e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c4.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| unique snapshot | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1750) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2626) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2520) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2358) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1817) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:608) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:566) | |
| org.opensearch.security.support.Base64Helper.deserializeObject(Base64Helper.java:257) | |
| org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:164) | |
| org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.TransportService.sendLocalRequest(TransportService.java:938) | |
| app//org.opensearch.transport.TransportService.access$100(TransportService.java:92) | |
| app//org.opensearch.transport.TransportService$3.sendRequest(TransportService.java:151) | |
| app//org.opensearch.transport.TransportService.sendRequestInternal(TransportService.java:876) | |
| app//org.opensearch.transport.TransportService$$Lambda$sendRequestInternal$2649957159/0x00000000000050f3.sendRequest(Unknown Source) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:207) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000684e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c4.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 9.8% (49.1ms out of 500ms) cpu usage by thread 'opensearch[datanode105.xyz.com][transport_worker][T#10]' | |
| 2/10 snapshots sharing following 44 elements | |
| org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:153) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.InboundHandler.handleRequest(InboundHandler.java:268) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:126) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005b80.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 7/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 7.9% (39.5ms out of 500ms) cpu usage by thread 'opensearch[datanode105.xyz.com][transport_worker][T#6]' | |
| 10/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode108.xyz.com}{-dUr4PyJQIixE17KOTDoPw}{GsiUAVUiRwWt5ieSs3FTPQ}{datanode108.xyz.com}{10.218.9.212:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:55.987Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 97.9% (489.3ms out of 500ms) cpu usage by thread 'opensearch[datanode108.xyz.com][transport_worker][T#7]' | |
| 3/10 snapshots sharing following 76 elements | |
| java.base@11.0.14.1.101/java.io.PrintStream.println(PrintStream.java:882) | |
| java.base@11.0.14.1.101/sun.security.util.Debug.println(Debug.java:184) | |
| java.base@11.0.14.1.101/java.security.AccessControlContext.checkPermission(AccessControlContext.java:478) | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:897) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.enableReplaceObject(ObjectOutputStream.java:623) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.lambda$new$1(Base64Helper.java:217) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream$$Lambda$lambda$new$1$486334844/0x00000000000004ff.run(Unknown Source) | |
| java.base@11.0.14.1.101/java.security.AccessController.doPrivileged(Native Method) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.<init>(Base64Helper.java:216) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006905.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x000000000000697b.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d52.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 3/10 snapshots sharing following 64 elements | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006905.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x000000000000697b.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d52.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 24 elements | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 17.3% (86.3ms out of 500ms) cpu usage by thread 'opensearch[datanode108.xyz.com][transport_worker][T#1]' | |
| 3/10 snapshots sharing following 50 elements | |
| java.base@11.0.14.1.101/java.io.PrintStream.println(PrintStream.java:882) | |
| java.base@11.0.14.1.101/sun.security.util.Debug.println(Debug.java:184) | |
| java.base@11.0.14.1.101/java.security.AccessControlContext.checkPermission(AccessControlContext.java:478) | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:897) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| org.opensearch.security.ssl.transport.DefaultPrincipalExtractor.extractPrincipal(DefaultPrincipalExtractor.java:50) | |
| org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:143) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.InboundHandler.handleRequest(InboundHandler.java:268) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:126) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d52.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 3/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 4/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 16.1% (80.3ms out of 500ms) cpu usage by thread 'opensearch[datanode108.xyz.com][transport_worker][T#5]' | |
| 2/10 snapshots sharing following 72 elements | |
| java.base@11.0.14.1.101/java.io.PrintStream.println(PrintStream.java:882) | |
| java.base@11.0.14.1.101/sun.security.util.Debug.println(Debug.java:184) | |
| java.base@11.0.14.1.101/java.security.AccessControlContext.checkPermission(AccessControlContext.java:478) | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:897) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.<init>(Base64Helper.java:213) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006905.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x000000000000697b.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d52.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 75 elements | |
| java.base@11.0.14.1.101/sun.security.util.Debug.println(Debug.java:184) | |
| java.base@11.0.14.1.101/java.security.AccessControlContext.checkPermission(AccessControlContext.java:478) | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:897) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.enableReplaceObject(ObjectOutputStream.java:623) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.lambda$new$1(Base64Helper.java:217) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream$$Lambda$lambda$new$1$486334844/0x00000000000004ff.run(Unknown Source) | |
| java.base@11.0.14.1.101/java.security.AccessController.doPrivileged(Native Method) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.<init>(Base64Helper.java:216) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006905.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x000000000000697b.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d52.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 67 elements | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006905.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x000000000000697b.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d52.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 30 elements | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 10 elements | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.flush(Netty4MessageChannelHandler.java:135) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:750) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:765) | |
| io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071) | |
| io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) | |
| io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:469) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode107.xyz.com}{VIusRcUdRoW7fDPV4ybdig}{7q0H0yXvQo2NYLe9eihLCw}{datanode107.xyz.com}{10.218.9.211:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:55.613Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 12.9% (64.3ms out of 500ms) cpu usage by thread 'opensearch[datanode107.xyz.com][transport_worker][T#4]' | |
| 6/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 10.0% (50.2ms out of 500ms) cpu usage by thread 'opensearch[datanode107.xyz.com][transport_worker][T#10]' | |
| 6/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode104.xyz.com}{7wGFZrOLSiCpjhSzFsq9bQ}{iX2EGtP7SnyWXnKvhJCFcQ}{datanode104.xyz.com}{10.218.9.208:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:55.973Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 9.5% (47.4ms out of 500ms) cpu usage by thread 'opensearch[datanode104.xyz.com][transport_worker][T#9]' | |
| 2/10 snapshots sharing following 43 elements | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.InboundHandler.handleRequest(InboundHandler.java:268) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:126) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005b80.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 8/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 9.4% (46.9ms out of 500ms) cpu usage by thread 'opensearch[datanode104.xyz.com][transport_worker][T#5]' | |
| 2/10 snapshots sharing following 43 elements | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.InboundHandler.handleRequest(InboundHandler.java:268) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:126) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005b80.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 8/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode110.xyz.com}{RDbmkqioTOqmgCCcOFtaCQ}{2e8szEuPRyStg-RmZN9tjw}{datanode110.xyz.com}{10.218.9.214:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:55.741Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 99.2% (496.1ms out of 500ms) cpu usage by thread 'opensearch[datanode110.xyz.com][transport_worker][T#2]' | |
| 2/10 snapshots sharing following 69 elements | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:897) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.<init>(Base64Helper.java:213) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x00000000000069b0.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x0000000000006a43.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d50.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 43 elements | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.InboundHandler.handleRequest(InboundHandler.java:268) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:126) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d50.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 6/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 12.0% (60.1ms out of 500ms) cpu usage by thread 'opensearch[datanode110.xyz.com][transport_worker][T#6]' | |
| 3/10 snapshots sharing following 63 elements | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x00000000000069b0.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x0000000000006a43.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d50.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 5/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode109.xyz.com}{SgHVvZR8RI6-pH6u6hbCjA}{CS9DAWzeTrGI9d0-bqfD_w}{datanode109.xyz.com}{10.218.9.213:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:58.793Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 27.7% (138.4ms out of 500ms) cpu usage by thread 'opensearch[datanode109.xyz.com][transport_worker][T#9]' | |
| 4/10 snapshots sharing following 89 elements | |
| org.opensearch.security.securityconf.ConfigModelV7$IndexPattern.access$400(ConfigModelV7.java:643) | |
| org.opensearch.security.securityconf.ConfigModelV7.lambda$impliesTypePerm$4(ConfigModelV7.java:965) | |
| org.opensearch.security.securityconf.ConfigModelV7$$Lambda$lambda$impliesTypePerm$4$590894108/0x0000000000006109.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195) | |
| java.base@11.0.14.1.101/java.util.HashMap$KeySpliterator.forEachRemaining(HashMap.java:1621) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:550) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.evaluateToArrayNode(AbstractPipeline.java:260) | |
| java.base@11.0.14.1.101/java.util.stream.ReferencePipeline.toArray(ReferencePipeline.java:517) | |
| org.opensearch.security.securityconf.ConfigModelV7.impliesTypePerm(ConfigModelV7.java:966) | |
| org.opensearch.security.securityconf.ConfigModelV7.access$600(ConfigModelV7.java:68) | |
| org.opensearch.security.securityconf.ConfigModelV7$SecurityRoles.impliesTypePermGlobal(ConfigModelV7.java:487) | |
| org.opensearch.security.privileges.PrivilegesEvaluator.evaluate(PrivilegesEvaluator.java:460) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:305) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003297.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 6/10 snapshots sharing following 87 elements | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x000000000000612c.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000003297.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 24.9% (124.7ms out of 500ms) cpu usage by thread 'opensearch[datanode109.xyz.com][transport_worker][T#7]' | |
| 9/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode102.xyz.com}{l7Euuj60RbqQTC1aavG6Ig}{2RtDXmisSZWRlss3EKafCQ}{datanode102.xyz.com}{10.218.9.206:50141}{dm}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:59.005Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 27.0% (135ms out of 500ms) cpu usage by thread 'opensearch[datanode102.xyz.com][transport_worker][T#4]' | |
| 6/10 snapshots sharing following 89 elements | |
| org.opensearch.security.securityconf.ConfigModelV7$IndexPattern.access$400(ConfigModelV7.java:643) | |
| org.opensearch.security.securityconf.ConfigModelV7.lambda$impliesTypePerm$4(ConfigModelV7.java:965) | |
| org.opensearch.security.securityconf.ConfigModelV7$$Lambda$lambda$impliesTypePerm$4$590894108/0x00000000000063a9.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195) | |
| java.base@11.0.14.1.101/java.util.HashMap$KeySpliterator.forEachRemaining(HashMap.java:1621) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:550) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.evaluateToArrayNode(AbstractPipeline.java:260) | |
| java.base@11.0.14.1.101/java.util.stream.ReferencePipeline.toArray(ReferencePipeline.java:517) | |
| org.opensearch.security.securityconf.ConfigModelV7.impliesTypePerm(ConfigModelV7.java:966) | |
| org.opensearch.security.securityconf.ConfigModelV7.access$600(ConfigModelV7.java:68) | |
| org.opensearch.security.securityconf.ConfigModelV7$SecurityRoles.impliesTypePermGlobal(ConfigModelV7.java:487) | |
| org.opensearch.security.privileges.PrivilegesEvaluator.evaluate(PrivilegesEvaluator.java:460) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:305) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005fd0.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 4/10 snapshots sharing following 87 elements | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000063f6.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005fd0.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 8.7% (43.7ms out of 500ms) cpu usage by thread 'opensearch[datanode102.xyz.com][transport_worker][T#2]' | |
| 6/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode101.xyz.com}{9IHR5tYxRNerG7R5OtxsvQ}{EGoQdq1oREmP_7NLJeaHiQ}{datanode101.xyz.com}{10.218.9.205:50141}{dm}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:59.169Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 24.3% (121.2ms out of 500ms) cpu usage by thread 'opensearch[datanode101.xyz.com][transport_worker][T#6]' | |
| 2/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode103.xyz.com}{Lvc0FiOTRlShBEDWa4XtQQ}{-mLQ-ZHbSgOyGS6ptZPLhQ}{datanode103.xyz.com}{10.218.9.207:50141}{dm}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:58.515Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 94.0% (470.1ms out of 500ms) cpu usage by thread 'opensearch[datanode103.xyz.com][transport_worker][T#10]' | |
| 3/10 snapshots sharing following 36 elements | |
| java.base@11.0.14.1.101/com.sun.crypto.provider.AESCipher.engineDoFinal(AESCipher.java:491) | |
| java.base@11.0.14.1.101/javax.crypto.CipherSpi.bufferCrypt(CipherSpi.java:779) | |
| java.base@11.0.14.1.101/javax.crypto.CipherSpi.engineDoFinal(CipherSpi.java:730) | |
| java.base@11.0.14.1.101/javax.crypto.Cipher.doFinal(Cipher.java:2497) | |
| java.base@11.0.14.1.101/sun.security.ssl.SSLCipher$T13GcmWriteCipherGenerator$GcmWriteCipher.encrypt(SSLCipher.java:2078) | |
| java.base@11.0.14.1.101/sun.security.ssl.OutputRecord.t13Encrypt(OutputRecord.java:413) | |
| java.base@11.0.14.1.101/sun.security.ssl.OutputRecord.encrypt(OutputRecord.java:336) | |
| java.base@11.0.14.1.101/sun.security.ssl.SSLEngineOutputRecord.encode(SSLEngineOutputRecord.java:278) | |
| java.base@11.0.14.1.101/sun.security.ssl.SSLEngineOutputRecord.encode(SSLEngineOutputRecord.java:180) | |
| java.base@11.0.14.1.101/sun.security.ssl.SSLEngineImpl.encode(SSLEngineImpl.java:285) | |
| java.base@11.0.14.1.101/sun.security.ssl.SSLEngineImpl.writeRecord(SSLEngineImpl.java:231) | |
| java.base@11.0.14.1.101/sun.security.ssl.SSLEngineImpl.wrap(SSLEngineImpl.java:136) | |
| java.base@11.0.14.1.101/sun.security.ssl.SSLEngineImpl.wrap(SSLEngineImpl.java:116) | |
| java.base@11.0.14.1.101/javax.net.ssl.SSLEngine.wrap(SSLEngine.java:522) | |
| io.netty.handler.ssl.SslHandler.wrap(SslHandler.java:1039) | |
| io.netty.handler.ssl.SslHandler.wrap(SslHandler.java:826) | |
| io.netty.handler.ssl.SslHandler.wrapAndFlush(SslHandler.java:797) | |
| io.netty.handler.ssl.SslHandler.flush(SslHandler.java:778) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:750) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:742) | |
| io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:728) | |
| io.netty.handler.logging.LoggingHandler.flush(LoggingHandler.java:304) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:750) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:742) | |
| io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:728) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.doFlush(Netty4MessageChannelHandler.java:200) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.flush(Netty4MessageChannelHandler.java:135) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:750) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:765) | |
| io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071) | |
| io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) | |
| io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:469) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:503) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 68 elements | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.<init>(Base64Helper.java:213) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006811.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x000000000000686b.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005b7a.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 24 elements | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| unique snapshot | |
| java.base@11.0.14.1.101/sun.nio.ch.FileDispatcherImpl.read0(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) | |
| java.base@11.0.14.1.101/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:276) | |
| java.base@11.0.14.1.101/sun.nio.ch.IOUtil.read(IOUtil.java:233) | |
| java.base@11.0.14.1.101/sun.nio.ch.IOUtil.read(IOUtil.java:223) | |
| java.base@11.0.14.1.101/sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:356) | |
| org.opensearch.transport.CopyBytesSocketChannel.readFromSocketChannel(CopyBytesSocketChannel.java:155) | |
| org.opensearch.transport.CopyBytesSocketChannel.doReadBytes(CopyBytesSocketChannel.java:140) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 28.2% (140.9ms out of 500ms) cpu usage by thread 'opensearch[datanode103.xyz.com][transport_worker][T#7]' | |
| 3/10 snapshots sharing following 63 elements | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006811.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x000000000000686b.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005b7a.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 7/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 25.6% (128.1ms out of 500ms) cpu usage by thread 'opensearch[datanode103.xyz.com][transport_worker][T#5]' | |
| 3/10 snapshots sharing following 24 elements | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 5/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode106.xyz.com}{XPP-bmQATm2ygjbpSUWuHQ}{kim-kJzoQLeWtV3htztH8g}{datanode106.xyz.com}{10.218.9.210:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:58.817Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 30.4% (152.1ms out of 500ms) cpu usage by thread 'opensearch[datanode106.xyz.com][transport_worker][T#8]' | |
| 3/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode105.xyz.com}{naQqgBCNREGHSOQlJ4-XaQ}{p9c9s3RMTkWPX_lY4fU7VA}{datanode105.xyz.com}{10.218.9.209:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:58.869Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 28.7% (143.6ms out of 500ms) cpu usage by thread 'opensearch[datanode105.xyz.com][transport_worker][T#2]' | |
| 4/10 snapshots sharing following 93 elements | |
| app//org.opensearch.cluster.metadata.IndexNameExpressionResolver.concreteIndices(IndexNameExpressionResolver.java:338) | |
| app//org.opensearch.cluster.metadata.IndexNameExpressionResolver.concreteIndexNames(IndexNameExpressionResolver.java:232) | |
| app//org.opensearch.cluster.metadata.IndexNameExpressionResolver.concreteIndexNames(IndexNameExpressionResolver.java:154) | |
| org.opensearch.security.securityconf.ConfigModelV7$IndexPattern.getResolvedIndexPattern(ConfigModelV7.java:754) | |
| org.opensearch.security.securityconf.ConfigModelV7$IndexPattern.access$400(ConfigModelV7.java:643) | |
| org.opensearch.security.securityconf.ConfigModelV7.lambda$impliesTypePerm$4(ConfigModelV7.java:965) | |
| org.opensearch.security.securityconf.ConfigModelV7$$Lambda$lambda$impliesTypePerm$4$590894108/0x00000000000067ab.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195) | |
| java.base@11.0.14.1.101/java.util.HashMap$KeySpliterator.forEachRemaining(HashMap.java:1621) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:550) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.evaluateToArrayNode(AbstractPipeline.java:260) | |
| java.base@11.0.14.1.101/java.util.stream.ReferencePipeline.toArray(ReferencePipeline.java:517) | |
| org.opensearch.security.securityconf.ConfigModelV7.impliesTypePerm(ConfigModelV7.java:966) | |
| org.opensearch.security.securityconf.ConfigModelV7.access$600(ConfigModelV7.java:68) | |
| org.opensearch.security.securityconf.ConfigModelV7$SecurityRoles.impliesTypePermGlobal(ConfigModelV7.java:487) | |
| org.opensearch.security.privileges.PrivilegesEvaluator.evaluate(PrivilegesEvaluator.java:460) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:305) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 90 elements | |
| org.opensearch.security.securityconf.ConfigModelV7$IndexPattern.getResolvedIndexPattern(ConfigModelV7.java:751) | |
| org.opensearch.security.securityconf.ConfigModelV7$IndexPattern.access$400(ConfigModelV7.java:643) | |
| org.opensearch.security.securityconf.ConfigModelV7.lambda$impliesTypePerm$4(ConfigModelV7.java:965) | |
| org.opensearch.security.securityconf.ConfigModelV7$$Lambda$lambda$impliesTypePerm$4$590894108/0x00000000000067ab.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195) | |
| java.base@11.0.14.1.101/java.util.HashMap$KeySpliterator.forEachRemaining(HashMap.java:1621) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:550) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.evaluateToArrayNode(AbstractPipeline.java:260) | |
| java.base@11.0.14.1.101/java.util.stream.ReferencePipeline.toArray(ReferencePipeline.java:517) | |
| org.opensearch.security.securityconf.ConfigModelV7.impliesTypePerm(ConfigModelV7.java:966) | |
| org.opensearch.security.securityconf.ConfigModelV7.access$600(ConfigModelV7.java:68) | |
| org.opensearch.security.securityconf.ConfigModelV7$SecurityRoles.impliesTypePermGlobal(ConfigModelV7.java:487) | |
| org.opensearch.security.privileges.PrivilegesEvaluator.evaluate(PrivilegesEvaluator.java:460) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:305) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 95 elements | |
| app//org.opensearch.cluster.metadata.IndexNameExpressionResolver$WildcardExpressionResolver.innerResolve(IndexNameExpressionResolver.java:963) | |
| app//org.opensearch.cluster.metadata.IndexNameExpressionResolver$WildcardExpressionResolver.resolve(IndexNameExpressionResolver.java:946) | |
| app//org.opensearch.cluster.metadata.IndexNameExpressionResolver.concreteIndices(IndexNameExpressionResolver.java:252) | |
| app//org.opensearch.cluster.metadata.IndexNameExpressionResolver.concreteIndexNames(IndexNameExpressionResolver.java:232) | |
| app//org.opensearch.cluster.metadata.IndexNameExpressionResolver.concreteIndexNames(IndexNameExpressionResolver.java:154) | |
| org.opensearch.security.securityconf.ConfigModelV7$IndexPattern.getResolvedIndexPattern(ConfigModelV7.java:754) | |
| org.opensearch.security.securityconf.ConfigModelV7$IndexPattern.access$400(ConfigModelV7.java:643) | |
| org.opensearch.security.securityconf.ConfigModelV7.lambda$impliesTypePerm$4(ConfigModelV7.java:965) | |
| org.opensearch.security.securityconf.ConfigModelV7$$Lambda$lambda$impliesTypePerm$4$590894108/0x00000000000067ab.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195) | |
| java.base@11.0.14.1.101/java.util.HashMap$KeySpliterator.forEachRemaining(HashMap.java:1621) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:550) | |
| java.base@11.0.14.1.101/java.util.stream.AbstractPipeline.evaluateToArrayNode(AbstractPipeline.java:260) | |
| java.base@11.0.14.1.101/java.util.stream.ReferencePipeline.toArray(ReferencePipeline.java:517) | |
| org.opensearch.security.securityconf.ConfigModelV7.impliesTypePerm(ConfigModelV7.java:966) | |
| org.opensearch.security.securityconf.ConfigModelV7.access$600(ConfigModelV7.java:68) | |
| org.opensearch.security.securityconf.ConfigModelV7$SecurityRoles.impliesTypePermGlobal(ConfigModelV7.java:487) | |
| org.opensearch.security.privileges.PrivilegesEvaluator.evaluate(PrivilegesEvaluator.java:460) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:305) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 74 elements | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 11.8% (59.2ms out of 500ms) cpu usage by thread 'opensearch[datanode105.xyz.com][transport_worker][T#6]' | |
| 7/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode108.xyz.com}{-dUr4PyJQIixE17KOTDoPw}{GsiUAVUiRwWt5ieSs3FTPQ}{datanode108.xyz.com}{10.218.9.212:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:59.257Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 34.3% (171.7ms out of 500ms) cpu usage by thread 'opensearch[datanode108.xyz.com][transport_worker][T#7]' | |
| 10/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 33.6% (168.1ms out of 500ms) cpu usage by thread 'opensearch[datanode108.xyz.com][transport_worker][T#9]' | |
| 2/10 snapshots sharing following 43 elements | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.InboundHandler.handleRequest(InboundHandler.java:268) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:126) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d52.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 6/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:817) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode107.xyz.com}{VIusRcUdRoW7fDPV4ybdig}{7q0H0yXvQo2NYLe9eihLCw}{datanode107.xyz.com}{10.218.9.211:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:58.883Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 100.4% (501.7ms out of 500ms) cpu usage by thread 'opensearch[datanode107.xyz.com][transport_worker][T#5]' | |
| 2/10 snapshots sharing following 40 elements | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:126) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d51.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 85 elements | |
| org.apache.commons.lang3.SerializationUtils$ClassLoaderAwareObjectInputStream.resolveClass(SerializationUtils.java:325) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:2133) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:2000) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1795) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:608) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:566) | |
| org.apache.commons.lang3.SerializationUtils.clone(SerializationUtils.java:95) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.lambda$replace$0(Base64Helper.java:164) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$0$883570508/0x0000000000006933.run(Unknown Source) | |
| java.base@11.0.14.1.101/java.security.AccessController.doPrivileged(Native Method) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.lambda$replace$1(Base64Helper.java:163) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$1$1358610711/0x0000000000006932.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.replace(Base64Helper.java:160) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.writeClassDescriptor(Base64Helper.java:223) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeNonProxyDesc(ObjectOutputStream.java:1283) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeClassDesc(ObjectOutputStream.java:1232) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1428) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000691e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x000000000000695f.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d51.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 5/10 snapshots sharing following 15 elements | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| unique snapshot | |
| java.base@11.0.14.1.101/sun.nio.ch.FileDispatcherImpl.read0(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) | |
| java.base@11.0.14.1.101/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:276) | |
| java.base@11.0.14.1.101/sun.nio.ch.IOUtil.read(IOUtil.java:233) | |
| java.base@11.0.14.1.101/sun.nio.ch.IOUtil.read(IOUtil.java:223) | |
| java.base@11.0.14.1.101/sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:356) | |
| org.opensearch.transport.CopyBytesSocketChannel.readFromSocketChannel(CopyBytesSocketChannel.java:155) | |
| org.opensearch.transport.CopyBytesSocketChannel.doReadBytes(CopyBytesSocketChannel.java:140) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 12.0% (60ms out of 500ms) cpu usage by thread 'opensearch[datanode107.xyz.com][transport_worker][T#10]' | |
| 2/10 snapshots sharing following 50 elements | |
| java.base@11.0.14.1.101/java.io.PrintStream.println(PrintStream.java:882) | |
| java.base@11.0.14.1.101/sun.security.util.Debug.println(Debug.java:184) | |
| java.base@11.0.14.1.101/java.security.AccessControlContext.checkPermission(AccessControlContext.java:478) | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:897) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| org.opensearch.security.ssl.transport.DefaultPrincipalExtractor.extractPrincipal(DefaultPrincipalExtractor.java:50) | |
| org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:143) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.InboundHandler.handleRequest(InboundHandler.java:268) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:126) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d51.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 76 elements | |
| java.base@11.0.14.1.101/java.io.PrintStream.println(PrintStream.java:882) | |
| java.base@11.0.14.1.101/sun.security.util.Debug.println(Debug.java:184) | |
| java.base@11.0.14.1.101/java.security.AccessControlContext.checkPermission(AccessControlContext.java:478) | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:897) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.enableReplaceObject(ObjectOutputStream.java:623) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.lambda$new$1(Base64Helper.java:217) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream$$Lambda$lambda$new$1$486334844/0x00000000000060af.run(Unknown Source) | |
| java.base@11.0.14.1.101/java.security.AccessController.doPrivileged(Native Method) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.<init>(Base64Helper.java:216) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000691e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x000000000000695f.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d51.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 5/10 snapshots sharing following 7 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| unique snapshot | |
| java.base@11.0.14.1.101/java.lang.Class.forName0(Native Method) | |
| java.base@11.0.14.1.101/java.lang.Class.forName(Class.java:398) | |
| org.apache.commons.lang3.SerializationUtils$ClassLoaderAwareObjectInputStream.resolveClass(SerializationUtils.java:322) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:2133) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:2000) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1795) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:608) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:566) | |
| org.apache.commons.lang3.SerializationUtils.clone(SerializationUtils.java:95) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.lambda$replace$0(Base64Helper.java:164) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$0$883570508/0x0000000000006933.run(Unknown Source) | |
| java.base@11.0.14.1.101/java.security.AccessController.doPrivileged(Native Method) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.lambda$replace$1(Base64Helper.java:163) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$1$1358610711/0x0000000000006932.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.replace(Base64Helper.java:160) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.writeClassDescriptor(Base64Helper.java:223) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeNonProxyDesc(ObjectOutputStream.java:1283) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeClassDesc(ObjectOutputStream.java:1232) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1428) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000691e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x000000000000695f.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d51.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 11.4% (56.8ms out of 500ms) cpu usage by thread 'opensearch[datanode107.xyz.com][transport_worker][T#9]' | |
| 3/10 snapshots sharing following 44 elements | |
| org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:153) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.InboundHandler.handleRequest(InboundHandler.java:268) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:126) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d51.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 46 elements | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| org.opensearch.security.ssl.transport.DefaultPrincipalExtractor.extractPrincipal(DefaultPrincipalExtractor.java:50) | |
| org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:143) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.InboundHandler.handleRequest(InboundHandler.java:268) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:126) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d51.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 3/10 snapshots sharing following 24 elements | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode104.xyz.com}{7wGFZrOLSiCpjhSzFsq9bQ}{iX2EGtP7SnyWXnKvhJCFcQ}{datanode104.xyz.com}{10.218.9.208:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:59.243Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 97.7% (488.3ms out of 500ms) cpu usage by thread 'opensearch[datanode104.xyz.com][transport_worker][T#3]' | |
| 2/10 snapshots sharing following 128 elements | |
| java.base@11.0.14.1.101/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:178) | |
| java.base@11.0.14.1.101/java.lang.ClassLoader.loadClass(ClassLoader.java:522) | |
| java.base@11.0.14.1.101/java.lang.Class.forName0(Native Method) | |
| java.base@11.0.14.1.101/java.lang.Class.forName(Class.java:398) | |
| org.apache.commons.lang3.SerializationUtils$ClassLoaderAwareObjectInputStream.resolveClass(SerializationUtils.java:325) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:2133) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:2000) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1795) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:608) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:566) | |
| org.apache.commons.lang3.SerializationUtils.clone(SerializationUtils.java:95) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.lambda$replace$0(Base64Helper.java:164) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$0$883570508/0x0000000000006847.run(Unknown Source) | |
| java.base@11.0.14.1.101/java.security.AccessController.doPrivileged(Native Method) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.lambda$replace$1(Base64Helper.java:163) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$1$1358610711/0x0000000000006834.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.replace(Base64Helper.java:160) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.writeClassDescriptor(Base64Helper.java:223) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeNonProxyDesc(ObjectOutputStream.java:1283) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeClassDesc(ObjectOutputStream.java:1232) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1428) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006829.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c9.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000006059.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 114 elements | |
| java.base@11.0.14.1.101/sun.security.util.Debug.println(Debug.java:184) | |
| java.base@11.0.14.1.101/java.security.AccessControlContext.checkPermission(AccessControlContext.java:478) | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:897) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.enableReplaceObject(ObjectOutputStream.java:623) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.lambda$new$1(Base64Helper.java:217) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream$$Lambda$lambda$new$1$486334844/0x0000000000005bdc.run(Unknown Source) | |
| java.base@11.0.14.1.101/java.security.AccessController.doPrivileged(Native Method) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.<init>(Base64Helper.java:216) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006829.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c9.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000006059.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 6/10 snapshots sharing following 101 elements | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006829.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c9.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000006059.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 13.4% (66.8ms out of 500ms) cpu usage by thread 'opensearch[datanode104.xyz.com][transport_worker][T#7]' | |
| 2/10 snapshots sharing following 24 elements | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 4/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 12.5% (62.4ms out of 500ms) cpu usage by thread 'opensearch[datanode104.xyz.com][transport_worker][T#2]' | |
| 2/10 snapshots sharing following 24 elements | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 8/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode110.xyz.com}{RDbmkqioTOqmgCCcOFtaCQ}{2e8szEuPRyStg-RmZN9tjw}{datanode110.xyz.com}{10.218.9.214:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:35:59.011Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 90.5% (452.7ms out of 500ms) cpu usage by thread 'opensearch[datanode110.xyz.com][transport_worker][T#6]' | |
| 6/10 snapshots sharing following 20 elements | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 4/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 35.1% (175.6ms out of 500ms) cpu usage by thread 'opensearch[datanode110.xyz.com][transport_worker][T#2]' | |
| 2/10 snapshots sharing following 24 elements | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 7/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 33.0% (164.8ms out of 500ms) cpu usage by thread 'opensearch[datanode110.xyz.com][transport_worker][T#5]' | |
| 2/10 snapshots sharing following 66 elements | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x00000000000069b0.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x0000000000006a43.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d50.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 62 elements | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x00000000000069b0.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x0000000000006a43.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d50.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 6/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode109.xyz.com}{SgHVvZR8RI6-pH6u6hbCjA}{CS9DAWzeTrGI9d0-bqfD_w}{datanode109.xyz.com}{10.218.9.213:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:36:01.612Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 16.9% (84.6ms out of 500ms) cpu usage by thread 'opensearch[datanode109.xyz.com][transport_worker][T#7]' | |
| 2/10 snapshots sharing following 49 elements | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1817) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:608) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:566) | |
| org.opensearch.security.support.Base64Helper.deserializeObject(Base64Helper.java:257) | |
| org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:256) | |
| org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:153) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.InboundHandler.handleRequest(InboundHandler.java:268) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:126) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d51.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 3/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 15.2% (75.7ms out of 500ms) cpu usage by thread 'opensearch[datanode109.xyz.com][transport_worker][T#4]' | |
| 7/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode102.xyz.com}{l7Euuj60RbqQTC1aavG6Ig}{2RtDXmisSZWRlss3EKafCQ}{datanode102.xyz.com}{10.218.9.206:50141}{dm}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:36:01.823Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 100.3% (501.3ms out of 500ms) cpu usage by thread 'opensearch[datanode102.xyz.com][transport_worker][T#6]' | |
| 2/10 snapshots sharing following 118 elements | |
| java.base@11.0.14.1.101/java.security.AccessController.getStackAccessControlContext(Native Method) | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:871) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| app//org.opensearch.SpecialPermission.check(SpecialPermission.java:102) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.lambda$replace$1(Base64Helper.java:161) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$1$1358610711/0x0000000000006696.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.replace(Base64Helper.java:160) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.writeClassDescriptor(Base64Helper.java:223) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeNonProxyDesc(ObjectOutputStream.java:1283) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeClassDesc(ObjectOutputStream.java:1232) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1428) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006669.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000063f6.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005fd0.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 3/10 snapshots sharing following 105 elements | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006669.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000063f6.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005fd0.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 128 elements | |
| java.base@11.0.14.1.101/java.lang.Throwable.fillInStackTrace(Native Method) | |
| java.base@11.0.14.1.101/java.lang.Throwable.fillInStackTrace(Throwable.java:787) | |
| java.base@11.0.14.1.101/java.lang.Throwable.<init>(Throwable.java:270) | |
| java.base@11.0.14.1.101/java.lang.Exception.<init>(Exception.java:66) | |
| java.base@11.0.14.1.101/java.io.IOException.<init>(IOException.java:58) | |
| java.base@11.0.14.1.101/java.io.PrintStream.ensureOpen(PrintStream.java:404) | |
| java.base@11.0.14.1.101/java.io.PrintStream.write(PrintStream.java:603) | |
| java.base@11.0.14.1.101/java.io.PrintStream.print(PrintStream.java:745) | |
| java.base@11.0.14.1.101/java.io.PrintStream.println(PrintStream.java:882) | |
| java.base@11.0.14.1.101/sun.security.util.Debug.println(Debug.java:184) | |
| java.base@11.0.14.1.101/java.security.AccessControlContext.checkPermission(AccessControlContext.java:478) | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:897) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| app//org.opensearch.SpecialPermission.check(SpecialPermission.java:102) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.lambda$replace$1(Base64Helper.java:161) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$1$1358610711/0x0000000000006696.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.replace(Base64Helper.java:160) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.writeClassDescriptor(Base64Helper.java:223) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeNonProxyDesc(ObjectOutputStream.java:1283) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeClassDesc(ObjectOutputStream.java:1232) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1428) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006669.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000063f6.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005fd0.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 105 elements | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006669.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000063f6.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005fd0.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| unique snapshot | |
| java.base@11.0.14.1.101/java.lang.Throwable.fillInStackTrace(Native Method) | |
| java.base@11.0.14.1.101/java.lang.Throwable.fillInStackTrace(Throwable.java:787) | |
| java.base@11.0.14.1.101/java.lang.Throwable.<init>(Throwable.java:270) | |
| java.base@11.0.14.1.101/java.lang.Exception.<init>(Exception.java:66) | |
| java.base@11.0.14.1.101/java.io.IOException.<init>(IOException.java:58) | |
| java.base@11.0.14.1.101/java.io.PrintStream.ensureOpen(PrintStream.java:404) | |
| java.base@11.0.14.1.101/java.io.PrintStream.write(PrintStream.java:603) | |
| java.base@11.0.14.1.101/java.io.PrintStream.print(PrintStream.java:745) | |
| java.base@11.0.14.1.101/java.io.PrintStream.println(PrintStream.java:882) | |
| java.base@11.0.14.1.101/sun.security.util.Debug.println(Debug.java:184) | |
| java.base@11.0.14.1.101/java.security.AccessControlContext.checkPermission(AccessControlContext.java:478) | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:897) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| java.base@11.0.14.1.101/java.lang.ClassLoader.checkClassLoaderPermission(ClassLoader.java:2060) | |
| java.base@11.0.14.1.101/java.lang.Thread.getContextClassLoader(Thread.java:1545) | |
| org.apache.commons.lang3.SerializationUtils$ClassLoaderAwareObjectInputStream.resolveClass(SerializationUtils.java:325) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:2133) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:2000) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1795) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:608) | |
| java.base@11.0.14.1.101/java.io.ObjectInputStream.readObject(ObjectInputStream.java:566) | |
| org.apache.commons.lang3.SerializationUtils.clone(SerializationUtils.java:95) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.lambda$replace$0(Base64Helper.java:164) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$0$883570508/0x000000000000669a.run(Unknown Source) | |
| java.base@11.0.14.1.101/java.security.AccessController.doPrivileged(Native Method) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.lambda$replace$1(Base64Helper.java:163) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$1$1358610711/0x0000000000006696.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.replace(Base64Helper.java:160) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.writeClassDescriptor(Base64Helper.java:223) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeNonProxyDesc(ObjectOutputStream.java:1283) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeClassDesc(ObjectOutputStream.java:1232) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1428) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006669.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000063f6.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005fd0.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 14.4% (72.1ms out of 500ms) cpu usage by thread 'opensearch[datanode102.xyz.com][transport_worker][T#1]' | |
| 2/10 snapshots sharing following 43 elements | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.InboundHandler.handleRequest(InboundHandler.java:268) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:126) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x00000000000059b8.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 6/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| unique snapshot | |
| java.base@11.0.14.1.101/com.sun.crypto.provider.CipherCore.init(CipherCore.java:500) | |
| java.base@11.0.14.1.101/com.sun.crypto.provider.AESCipher.engineInit(AESCipher.java:346) | |
| java.base@11.0.14.1.101/javax.crypto.Cipher.init(Cipher.java:1431) | |
| java.base@11.0.14.1.101/sun.security.ssl.SSLCipher$T13GcmReadCipherGenerator$GcmReadCipher.decrypt(SSLCipher.java:1912) | |
| java.base@11.0.14.1.101/sun.security.ssl.SSLEngineInputRecord.decodeInputRecord(SSLEngineInputRecord.java:240) | |
| java.base@11.0.14.1.101/sun.security.ssl.SSLEngineInputRecord.decode(SSLEngineInputRecord.java:197) | |
| java.base@11.0.14.1.101/sun.security.ssl.SSLEngineInputRecord.decode(SSLEngineInputRecord.java:160) | |
| java.base@11.0.14.1.101/sun.security.ssl.SSLTransport.decode(SSLTransport.java:111) | |
| java.base@11.0.14.1.101/sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:681) | |
| java.base@11.0.14.1.101/sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:636) | |
| java.base@11.0.14.1.101/sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:454) | |
| java.base@11.0.14.1.101/sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:433) | |
| java.base@11.0.14.1.101/javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:637) | |
| io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:295) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1341) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 11.5% (57.4ms out of 500ms) cpu usage by thread 'opensearch[datanode102.xyz.com][transport_worker][T#10]' | |
| 5/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode101.xyz.com}{9IHR5tYxRNerG7R5OtxsvQ}{EGoQdq1oREmP_7NLJeaHiQ}{datanode101.xyz.com}{10.218.9.205:50141}{dm}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:36:01.988Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 11.3% (56.3ms out of 500ms) cpu usage by thread 'opensearch[datanode101.xyz.com][transport_worker][T#8]' | |
| 9/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 9.4% (47ms out of 500ms) cpu usage by thread 'opensearch[datanode101.xyz.com][transport_worker][T#7]' | |
| 6/10 snapshots sharing following 20 elements | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| unique snapshot | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:817) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode103.xyz.com}{Lvc0FiOTRlShBEDWa4XtQQ}{-mLQ-ZHbSgOyGS6ptZPLhQ}{datanode103.xyz.com}{10.218.9.207:50141}{dm}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:36:01.334Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 90.5% (452.2ms out of 500ms) cpu usage by thread 'opensearch[datanode103.xyz.com][transport_worker][T#3]' | |
| 2/10 snapshots sharing following 47 elements | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:897) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| org.opensearch.security.ssl.transport.DefaultPrincipalExtractor.extractPrincipal(DefaultPrincipalExtractor.java:50) | |
| org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:143) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) | |
| app//org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:91) | |
| app//org.opensearch.transport.InboundHandler.handleRequest(InboundHandler.java:268) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:126) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005b7a.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 4/10 snapshots sharing following 20 elements | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 4/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 24.6% (122.8ms out of 500ms) cpu usage by thread 'opensearch[datanode103.xyz.com][transport_worker][T#10]' | |
| 2/10 snapshots sharing following 66 elements | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006811.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x000000000000686b.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005b7a.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 68 elements | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.<init>(Base64Helper.java:213) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x0000000000006811.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x000000000000686b.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005b7a.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 6/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 22.4% (111.7ms out of 500ms) cpu usage by thread 'opensearch[datanode103.xyz.com][transport_worker][T#1]' | |
| 3/10 snapshots sharing following 24 elements | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 7/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode106.xyz.com}{XPP-bmQATm2ygjbpSUWuHQ}{kim-kJzoQLeWtV3htztH8g}{datanode106.xyz.com}{10.218.9.210:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:36:01.638Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 35.8% (179.2ms out of 500ms) cpu usage by thread 'opensearch[datanode106.xyz.com][transport_worker][T#8]' | |
| 5/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 32.7% (163.6ms out of 500ms) cpu usage by thread 'opensearch[datanode106.xyz.com][transport_worker][T#2]' | |
| 2/10 snapshots sharing following 15 elements | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 6/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode105.xyz.com}{naQqgBCNREGHSOQlJ4-XaQ}{p9c9s3RMTkWPX_lY4fU7VA}{datanode105.xyz.com}{10.218.9.209:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:36:01.689Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 85.9% (429.6ms out of 500ms) cpu usage by thread 'opensearch[datanode105.xyz.com][transport_worker][T#3]' | |
| 2/10 snapshots sharing following 117 elements | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:897) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| app//org.opensearch.SpecialPermission.check(SpecialPermission.java:102) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.lambda$replace$1(Base64Helper.java:161) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer$$Lambda$lambda$replace$1$1358610711/0x0000000000006859.apply(Unknown Source) | |
| java.base@11.0.14.1.101/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705) | |
| org.opensearch.security.support.Base64Helper$DescriptorReplacer.replace(Base64Helper.java:160) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.writeClassDescriptor(Base64Helper.java:223) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeNonProxyDesc(ObjectOutputStream.java:1283) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeClassDesc(ObjectOutputStream.java:1232) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1428) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1179) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:242) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000684e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c4.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 3/10 snapshots sharing following 107 elements | |
| java.base@11.0.14.1.101/java.security.AccessController.doPrivileged(Native Method) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.<init>(Base64Helper.java:216) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000684e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c4.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 3/10 snapshots sharing following 103 elements | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000684e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c4.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 110 elements | |
| java.base@11.0.14.1.101/sun.security.util.Debug.println(Debug.java:184) | |
| java.base@11.0.14.1.101/java.security.AccessControlContext.checkPermission(AccessControlContext.java:478) | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:897) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.<init>(Base64Helper.java:213) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000684e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:310) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:249) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:415) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:215) | |
| app//org.opensearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:994) | |
| app//org.opensearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:757) | |
| app//org.opensearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:398) | |
| app//org.opensearch.action.search.TransportSearchAction$$Lambda$lambda$executeRequest$3$2087405313/0x00000000000067c4.accept(Unknown Source) | |
| app//org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:78) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:136) | |
| app//org.opensearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:101) | |
| app//org.opensearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:277) | |
| app//org.opensearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:120) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:194) | |
| org.opensearch.indexmanagement.rollup.actionfilter.FieldCapsFilter.apply(FieldCapsFilter.kt:141) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| org.opensearch.security.filter.SecurityFilter.apply0(SecurityFilter.java:319) | |
| org.opensearch.security.filter.SecurityFilter.apply(SecurityFilter.java:154) | |
| app//org.opensearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:192) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:169) | |
| app//org.opensearch.action.support.TransportAction.execute(TransportAction.java:97) | |
| app//org.opensearch.client.node.NodeClient.executeLocally(NodeClient.java:108) | |
| app//org.opensearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:104) | |
| app//org.opensearch.client.support.AbstractClient.execute(AbstractClient.java:433) | |
| app//org.opensearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:137) | |
| app//org.opensearch.rest.action.search.RestSearchAction$$Lambda$lambda$prepareRequest$2$204398087/0x0000000000005f60.accept(Unknown Source) | |
| app//org.opensearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:128) | |
| org.opensearch.security.filter.SecurityRestFilter$1.handleRequest(SecurityRestFilter.java:126) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:306) | |
| app//org.opensearch.rest.RestController.tryAllHandlers(RestController.java:392) | |
| app//org.opensearch.rest.RestController.dispatchRequest(RestController.java:235) | |
| org.opensearch.security.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) | |
| app//org.opensearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:361) | |
| app//org.opensearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:440) | |
| app//org.opensearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:351) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:55) | |
| org.opensearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:41) | |
| io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| org.opensearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:71) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 13.2% (65.9ms out of 500ms) cpu usage by thread 'opensearch[datanode105.xyz.com][transport_worker][T#10]' | |
| 2/10 snapshots sharing following 19 elements | |
| io.netty.handler.ssl.SslHandler.flush(SslHandler.java:778) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:750) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:742) | |
| io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:728) | |
| io.netty.handler.logging.LoggingHandler.flush(LoggingHandler.java:304) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:750) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:742) | |
| io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:728) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.doFlush(Netty4MessageChannelHandler.java:200) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.flush(Netty4MessageChannelHandler.java:135) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:750) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:765) | |
| io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071) | |
| io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) | |
| io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:469) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:503) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 7/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode108.xyz.com}{-dUr4PyJQIixE17KOTDoPw}{GsiUAVUiRwWt5ieSs3FTPQ}{datanode108.xyz.com}{10.218.9.212:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:36:02.078Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 26.5% (132.6ms out of 500ms) cpu usage by thread 'opensearch[datanode108.xyz.com][transport_worker][T#1]' | |
| 2/10 snapshots sharing following 40 elements | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:126) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d52.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 8/10 snapshots sharing following 9 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.EPoll.wait(Native Method) | |
| java.base@11.0.14.1.101/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 20.2% (101ms out of 500ms) cpu usage by thread 'opensearch[datanode108.xyz.com][transport_worker][T#2]' | |
| 4/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| ::: {datanode107.xyz.com}{VIusRcUdRoW7fDPV4ybdig}{7q0H0yXvQo2NYLe9eihLCw}{datanode107.xyz.com}{10.218.9.211:50141}{d}{shard_indexing_pressure_enabled=true} | |
| Hot threads at 2022-08-31T09:36:01.704Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 97.7% (488.6ms out of 500ms) cpu usage by thread 'opensearch[datanode107.xyz.com][transport_worker][T#7]' | |
| 9/10 snapshots sharing following 3 elements | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 21.1% (105.3ms out of 500ms) cpu usage by thread 'opensearch[datanode107.xyz.com][transport_worker][T#5]' | |
| 2/10 snapshots sharing following 76 elements | |
| java.base@11.0.14.1.101/java.io.PrintStream.println(PrintStream.java:882) | |
| java.base@11.0.14.1.101/sun.security.util.Debug.println(Debug.java:184) | |
| java.base@11.0.14.1.101/java.security.AccessControlContext.checkPermission(AccessControlContext.java:478) | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:897) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| java.base@11.0.14.1.101/java.io.ObjectOutputStream.enableReplaceObject(ObjectOutputStream.java:623) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.lambda$new$1(Base64Helper.java:217) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream$$Lambda$lambda$new$1$486334844/0x00000000000060af.run(Unknown Source) | |
| java.base@11.0.14.1.101/java.security.AccessController.doPrivileged(Native Method) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.<init>(Base64Helper.java:216) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:228) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000691e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x000000000000695f.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d51.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 2/10 snapshots sharing following 64 elements | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000691e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x000000000000695f.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d51.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1371) | |
| io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1234) | |
| io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1283) | |
| io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507) | |
| io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446) | |
| io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) | |
| io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) | |
| io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) | |
| io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) | |
| io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 5/10 snapshots sharing following 7 elements | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) | |
| java.base@11.0.14.1.101/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) | |
| io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| unique snapshot | |
| com.appdynamics.appagent/com.singularity.ee.agent.util.reflect.ReflectionUtility.getDeclaredInterfaceNames(ReflectionUtility.java:371) | |
| com.appdynamics.appagent/com.singularity.ee.agent.util.reflect.ReflectionUtility.declaresInterfaceRecursively(ReflectionUtility.java:340) | |
| com.appdynamics.appagent/com.singularity.ee.agent.appagent.services.transactionmonitor.common.InstrumentationPointUtil.matchClass(InstrumentationPointUtil.java:486) | |
| com.appdynamics.appagent/com.singularity.ee.agent.appagent.services.transactionmonitor.common.InstrumentationPointUtil.matchClass(InstrumentationPointUtil.java:467) | |
| com.appdynamics.appagent/com.singularity.ee.agent.appagent.services.transactionmonitor.common.activity.CustomActivityUtil.isValid(CustomActivityUtil.java:345) | |
| com.appdynamics.appagent/com.singularity.ee.agent.appagent.services.transactionmonitor.common.activity.ActivityInProcessProducerInterceptor.setUpInProcessCorrelationInfo(ActivityInProcessProducerInterceptor.java:81) | |
| com.appdynamics.appagent/com.singularity.ee.agent.appagent.services.transactionmonitor.AInProcessProducerInterceptor.onMethodEndTracked(AInProcessProducerInterceptor.java:48) | |
| com.appdynamics.appagent/com.singularity.ee.agent.appagent.services.transactionmonitor.netty.v4.NettyV4ExitProducerInterceptor.onMethodEndTracked(NettyV4ExitProducerInterceptor.java:110) | |
| com.appdynamics.appagent/com.singularity.ee.agent.appagent.services.bciengine.AFastTrackedMethodInterceptor.onMethodEnd(AFastTrackedMethodInterceptor.java:83) | |
| com.appdynamics.appagent/com.singularity.ee.agent.appagent.kernel.bootimpl.FastMethodInterceptorDelegatorImpl.safeOnMethodEndNoReentrantCheck(FastMethodInterceptorDelegatorImpl.java:503) | |
| com.appdynamics.appagent/com.singularity.ee.agent.appagent.kernel.bootimpl.FastMethodInterceptorDelegatorImpl.safeOnMethodEnd(FastMethodInterceptorDelegatorImpl.java:429) | |
| com.singularity.ee.agent.appagent.entrypoint.bciengine.FastMethodInterceptorDelegatorBoot.safeOnMethodEnd(FastMethodInterceptorDelegatorBoot.java:119) | |
| com.singularity.ee.agent.appagent.entrypoint.bciengine.FastMethodInterceptorDelegatorBoot.safeOnMethodEndNormal(FastMethodInterceptorDelegatorBoot.java:97) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:721) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:709) | |
| io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:792) | |
| io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:702) | |
| io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:697) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.doFlush(Netty4MessageChannelHandler.java:181) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.flush(Netty4MessageChannelHandler.java:135) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:750) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:765) | |
| io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071) | |
| io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) | |
| io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:469) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) | |
| io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) | |
| java.base@11.0.14.1.101/java.lang.Thread.run(Thread.java:887) | |
| 17.5% (87.4ms out of 500ms) cpu usage by thread 'opensearch[datanode107.xyz.com][transport_worker][T#4]' | |
| 2/10 snapshots sharing following 69 elements | |
| java.base@11.0.14.1.101/java.security.AccessController.checkPermission(AccessController.java:897) | |
| java.base@11.0.14.1.101/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.<init>(Base64Helper.java:213) | |
| org.opensearch.security.support.Base64Helper$SafeObjectOutputStream.create(Base64Helper.java:199) | |
| org.opensearch.security.support.Base64Helper.serializeObject(Base64Helper.java:241) | |
| org.opensearch.security.transport.SecurityInterceptor.ensureCorrectHeaders(SecurityInterceptor.java:237) | |
| org.opensearch.security.transport.SecurityInterceptor.sendRequestDecorate(SecurityInterceptor.java:201) | |
| org.opensearch.security.OpenSearchSecurityPlugin$7$2.sendRequest(OpenSearchSecurityPlugin.java:661) | |
| app//org.opensearch.transport.TransportService.sendRequest(TransportService.java:763) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:838) | |
| app//org.opensearch.transport.TransportService.sendChildRequest(TransportService.java:826) | |
| app//org.opensearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:196) | |
| app//org.opensearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:124) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:278) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$lambda$performPhaseOnShard$3$2415975905/0x000000000000691e.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.tryRun(AbstractSearchAsyncAction.java:773) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$PendingExecutions.finishAndRunNext(AbstractSearchAsyncAction.java:767) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$$Lambda$finishAndRunNext$2509587942/0x000000000000695f.run(Unknown Source) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:745) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.executeNext(AbstractSearchAsyncAction.java:737) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:82) | |
| app//org.opensearch.action.search.AbstractSearchAsyncAction$1.innerOnResponse(AbstractSearchAsyncAction.java:284) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:57) | |
| app//org.opensearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:42) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:79) | |
| app//org.opensearch.action.search.SearchExecutionStatsCollector.onResponse(SearchExecutionStatsCollector.java:49) | |
| app//org.opensearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:67) | |
| app//org.opensearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:584) | |
| app//org.opensearch.transport.TransportService$6.handleResponse(TransportService.java:736) | |
| org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleResponse(SecurityInterceptor.java:302) | |
| app//org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1347) | |
| app//org.opensearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:319) | |
| app//org.opensearch.transport.InboundHandler.handleResponse(InboundHandler.java:311) | |
| app//org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:154) | |
| app//org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:108) | |
| app//org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:759) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$inboundMessage$2454171381/0x0000000000005d51.accept(Unknown Source) | |
| app//org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:170) | |
| app//org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:145) | |
| app//org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:110) | |
| org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:94) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) | |
| io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(Abstr |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment