Skip to content

Instantly share code, notes, and snippets.

@chrisxaustin
Created May 23, 2023 18:13
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save chrisxaustin/9befa14a1c14d0ba33873700830d06e1 to your computer and use it in GitHub Desktop.
Save chrisxaustin/9befa14a1c14d0ba33873700830d06e1 to your computer and use it in GitHub Desktop.
::: {alerts-es-hot-4}{-EeyMsEySdWaOrqgjuNvWg}{zxIWclJrTv6cLY8eqPrRYQ}{172.16.19.193}{172.16.19.193:9300}{hir}{k8s_node_name=ip-172-16-19-230.us-east-2.compute.internal, xpack.installed=true, data=hot, transform.node=false}
Hot threads at 2023-05-23T17:43:16.187Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
61.5% [cpu=61.5%, other=0.0%] (307.6ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-4][[alerts-redacted-traffic-000052][0]: Lucene Merge Thread #218]'
2/10 snapshots sharing following 15 elements
app//org.apache.lucene.index.MultiTermsEnum.postings(MultiTermsEnum.java:359)
app//org.apache.lucene.index.MappedMultiFields$MappedMultiTermsEnum.postings(MappedMultiFields.java:127)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:127)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 13 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:957)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 14 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:628)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:947)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 18 elements
app//org.apache.lucene.util.compress.LZ4$HighCompressionHashTable.get(LZ4.java:357)
app//org.apache.lucene.util.compress.LZ4.compressWithDictionary(LZ4.java:457)
app//org.apache.lucene.util.compress.LZ4.compress(LZ4.java:419)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlock(BlockTreeTermsWriter.java:823)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:606)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:947)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 13 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:947)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
32.7% [cpu=32.7%, other=0.0%] (163.3ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-4][write][T#8]'
2/10 snapshots sharing following 26 elements
app//org.elasticsearch.index.mapper.FieldMapper$MultiFields.parse(FieldMapper.java:519)
app//org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:259)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:577)
app//org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:741)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7137/0x00000008020d2240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 31 elements
app//org.apache.lucene.analysis.CharacterUtils.toLowerCase(CharacterUtils.java:58)
app//org.apache.lucene.analysis.LowerCaseFilter.incrementToken(LowerCaseFilter.java:42)
app//org.apache.lucene.analysis.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:49)
app//org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:924)
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7137/0x00000008020d2240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
4/10 snapshots sharing following 25 elements
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7137/0x00000008020d2240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 15 elements
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7137/0x00000008020d2240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
9.5% [cpu=9.5%, other=0.0%] (47.5ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-4][write][T#5]'
2/10 snapshots sharing following 29 elements
app//org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:182)
app//org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:974)
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7137/0x00000008020d2240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
5/10 snapshots sharing following 26 elements
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7137/0x00000008020d2240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 24 elements
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:577)
app//org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:741)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7137/0x00000008020d2240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
java.base@20.0.1/java.util.Objects.equals(Objects.java:64)
app//org.elasticsearch.index.mapper.DocumentParser.validateType(DocumentParser.java:160)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:77)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7137/0x00000008020d2240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
1.3% [cpu=1.3%, other=0.0%] (6.4ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-4][[alerts-redacted-000180][0]: Lucene Merge Thread #296]'
10/10 snapshots sharing following 17 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:97)
app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5313)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4821)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
1.1% [cpu=1.1%, other=0.0%] (5.3ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-4][[alerts-redacted-000074][0]: Lucene Merge Thread #277]'
10/10 snapshots sharing following 17 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:97)
app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5313)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4821)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
1.0% [cpu=1.0%, other=0.0%] (5.2ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-4][[alerts-redacted-traffic-000007][0]: Lucene Merge Thread #428]'
10/10 snapshots sharing following 11 elements
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.copyChunks(CompressingStoredFieldsWriter.java:583)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:633)
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:105)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
0.1% [cpu=0.1%, other=0.0%] (476.8micros out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-4][management][T#3]'
5/10 snapshots sharing following 3 elements
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (75.6micros out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-4][refresh][T#1]'
2/10 snapshots sharing following 27 elements
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
6/10 snapshots sharing following 10 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:676)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:616)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1294)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (49.2micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-4][write][T#3]'
6/10 snapshots sharing following 24 elements
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 17 elements
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-4][write][T#6]'
2/10 snapshots sharing following 21 elements
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7137/0x00000008020d2240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 27 elements
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7137/0x00000008020d2240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:525)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7137/0x00000008020d2240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'java-sdk-http-connection-reaper'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:188)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 5 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'ml-cpp-log-tail-thread'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.io.FileInputStream.readBytes(Native Method)
java.base@20.0.1/java.io.FileInputStream.read(FileInputStream.java:269)
org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:106)
org.elasticsearch.xpack.ml.process.NativeController.lambda$tailLogsInThread$0(NativeController.java:103)
org.elasticsearch.xpack.ml.process.NativeController$$Lambda$3538/0x000000080183fcd0.run(Unknown Source)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'process reaper (pid 214)'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.lang.ProcessHandleImpl.waitForProcessExit0(Native Method)
java.base@20.0.1/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:163)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
java.base@20.0.1/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:186)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.17.10]'
10/10 snapshots sharing following 8 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:221)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:754)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1099)
java.base@20.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
::: {alerts-es-hot-5}{PcNk8skgQwyN6dc2YZRoOA}{PPqENqJ5QW2Wppxvs-KYgg}{172.16.12.17}{172.16.12.17:9300}{hir}{k8s_node_name=ip-172-16-53-249.us-east-2.compute.internal, xpack.installed=true, data=hot, transform.node=false}
Hot threads at 2023-05-23T17:43:16.187Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
100.0% [cpu=71.6%, other=28.4%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-5][[alerts-redacted-traffic-000012][0]: Lucene Merge Thread #179]'
2/10 snapshots sharing following 13 elements
app//org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
app//org.apache.lucene.index.FilterLeafReader$FilterTermsEnum.next(FilterLeafReader.java:194)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:310)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
5/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
3/10 snapshots sharing following 13 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:947)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
100.0% [cpu=64.1%, other=35.9%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-5][[alerts-redacted-000087][0]: Lucene Merge Thread #638]'
2/10 snapshots sharing following 13 elements
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:161)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 16 elements
app//org.apache.lucene.codecs.MultiLevelSkipListWriter.bufferSkip(MultiLevelSkipListWriter.java:144)
app//org.apache.lucene.codecs.lucene84.Lucene84SkipWriter.bufferSkip(Lucene84SkipWriter.java:162)
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.startDoc(Lucene84PostingsWriter.java:223)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 14 elements
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.startDoc(Lucene84PostingsWriter.java:259)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 16 elements
app//org.apache.lucene.index.DocIDMerger$Sub.nextMappedDoc(DocIDMerger.java:56)
app//org.apache.lucene.index.DocIDMerger$SequentialDocIDMerger.next(DocIDMerger.java:116)
app//org.apache.lucene.index.MappingMultiPostingsEnum.nextDoc(MappingMultiPostingsEnum.java:103)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:133)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
100.0% [cpu=2.2%, other=97.8%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-5][transport_worker][T#1]'
10/10 snapshots sharing following 4 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=2.0%, other=98.0%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-5][transport_worker][T#7]'
10/10 snapshots sharing following 4 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
96.4% [cpu=80.5%, other=15.9%] (482ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-5][write][T#4]'
2/10 snapshots sharing following 28 elements
app//org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:924)
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7197/0x00000008020b2f60.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 15 elements
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7197/0x00000008020b2f60.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 23 elements
app//org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:844)
app//org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:865)
app//org.elasticsearch.index.engine.InternalEngine.ensureTranslogSynced(InternalEngine.java:569)
app//org.elasticsearch.index.shard.IndexShard$7.write(IndexShard.java:3743)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.processList(AsyncIOProcessor.java:97)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.drainAndProcessAndRelease(AsyncIOProcessor.java:85)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.put(AsyncIOProcessor.java:73)
app//org.elasticsearch.index.shard.IndexShard.sync(IndexShard.java:3766)
app//org.elasticsearch.action.support.replication.TransportWriteAction$AsyncAfterWriteAction.run(TransportWriteAction.java:455)
app//org.elasticsearch.action.support.replication.TransportWriteAction$WriteReplicaResult.runPostReplicaActions(TransportWriteAction.java:336)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.lambda$onResponse$2(TransportReplicationAction.java:669)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction$$Lambda$7164/0x00000008020ab840.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:447)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
4/10 snapshots sharing following 14 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue$Node.block(LinkedTransferQueue.java:470)
java.base@20.0.1/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
java.base@20.0.1/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:669)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:616)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1286)
app//org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:152)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
91.4% [cpu=70.1%, other=21.3%] (457ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-5][[alerts-redacted-traffic-000206][3]: Lucene Merge Thread #108]'
2/10 snapshots sharing following 17 elements
app//org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:212)
app//org.apache.lucene.codecs.lucene84.ForUtil.encode(ForUtil.java:291)
app//org.apache.lucene.codecs.lucene84.ForDeltaUtil.encodeDeltas(ForDeltaUtil.java:67)
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.startDoc(Lucene84PostingsWriter.java:243)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 16 elements
app//org.apache.lucene.codecs.lucene84.ForUtil.encode(ForUtil.java:291)
app//org.apache.lucene.codecs.lucene84.PForUtil.encode(PForUtil.java:93)
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.addPosition(Lucene84PostingsWriter.java:310)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:161)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
3/10 snapshots sharing following 16 elements
app//org.apache.lucene.index.DocIDMerger$Sub.nextMappedDoc(DocIDMerger.java:56)
app//org.apache.lucene.index.DocIDMerger$SequentialDocIDMerger.next(DocIDMerger.java:116)
app//org.apache.lucene.index.MappingMultiPostingsEnum.nextDoc(MappingMultiPostingsEnum.java:103)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:133)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 13 elements
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
unique snapshot
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:161)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
89.6% [cpu=57.6%, other=32.0%] (448ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-5][[alerts-redacted-000121][1]: Lucene Merge Thread #280]'
2/10 snapshots sharing following 13 elements
app//org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
app//org.apache.lucene.index.FilterLeafReader$FilterTermsEnum.next(FilterLeafReader.java:194)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:310)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 15 elements
app//org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:428)
app//org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
app//org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:307)
app//org.apache.lucene.index.FilterLeafReader$FilterTermsEnum.next(FilterLeafReader.java:194)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:310)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 13 elements
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:133)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
63.8% [cpu=31.8%, other=32.0%] (319ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-5][write][T#7]'
2/10 snapshots sharing following 31 elements
app//org.apache.lucene.analysis.standard.StandardTokenizer.incrementToken(StandardTokenizer.java:148)
app//org.apache.lucene.analysis.LowerCaseFilter.incrementToken(LowerCaseFilter.java:41)
app//org.apache.lucene.analysis.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:49)
app//org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:924)
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7197/0x00000008020b2f60.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 34 elements
java.base@20.0.1/jdk.internal.util.ArraysSupport.mismatch(ArraysSupport.java:298)
java.base@20.0.1/java.util.Arrays.equals(Arrays.java:2710)
app//org.apache.lucene.util.BytesRefHash.equals(BytesRefHash.java:202)
app//org.apache.lucene.util.BytesRefHash.findHash(BytesRefHash.java:348)
app//org.apache.lucene.util.BytesRefHash.add(BytesRefHash.java:273)
app//org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:182)
app//org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:974)
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7197/0x00000008020b2f60.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 26 elements
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7197/0x00000008020b2f60.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 29 elements
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:172)
app//org.apache.lucene.index.StoredFieldsConsumer.finishDocument(StoredFieldsConsumer.java:68)
app//org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:463)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:499)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7197/0x00000008020b2f60.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
60.0% [cpu=28.9%, other=31.1%] (300ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-5][write][T#2]'
2/10 snapshots sharing following 29 elements
app//org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:229)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:577)
app//org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:741)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:574)
app//org.elasticsearch.index.mapper.DocumentParser.parseObject(DocumentParser.java:608)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:502)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7197/0x00000008020b2f60.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 25 elements
app//org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:229)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:577)
app//org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:741)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7197/0x00000008020b2f60.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 28 elements
app//org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:974)
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7197/0x00000008020b2f60.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 26 elements
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7197/0x00000008020b2f60.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.findName(UTF8StreamJsonParser.java:2359)
app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseLongName(UTF8StreamJsonParser.java:1948)
app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseMediumName2(UTF8StreamJsonParser.java:1930)
app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseMediumName(UTF8StreamJsonParser.java:1889)
app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser._parseName(UTF8StreamJsonParser.java:1824)
app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextToken(UTF8StreamJsonParser.java:797)
app//org.elasticsearch.xcontent.json.JsonXContentParser.nextToken(JsonXContentParser.java:73)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:518)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7197/0x00000008020b2f60.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
53.0% [cpu=43.8%, other=9.2%] (265ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-5][refresh][T#4]'
2/10 snapshots sharing following 38 elements
app//org.apache.lucene.codecs.lucene80.IndexedDISI.advanceExact(IndexedDISI.java:399)
app//org.apache.lucene.codecs.lucene80.Lucene80NormsProducer$SparseNormsIterator.advanceExact(Lucene80NormsProducer.java:186)
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.startDoc(Lucene84PostingsWriter.java:259)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:120)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:267)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 34 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:120)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:267)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 35 elements
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:133)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:120)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:267)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 38 elements
app//org.apache.lucene.codecs.lucene84.ForUtil.encode(ForUtil.java:291)
app//org.apache.lucene.codecs.lucene84.PForUtil.encode(PForUtil.java:93)
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.addPosition(Lucene84PostingsWriter.java:310)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:161)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:120)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:267)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 34 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:120)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:267)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
44.6% [cpu=35.9%, other=8.7%] (223ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-5][[alerts-redacted-000004][0]: Lucene Merge Thread #120]'
3/10 snapshots sharing following 21 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.ByteBuffersDataOutput.copyTo(ByteBuffersDataOutput.java:287)
app//org.apache.lucene.util.bkd.BKDWriter$OneDimensionBKDWriter.writeLeafBlock(BKDWriter.java:677)
app//org.apache.lucene.util.bkd.BKDWriter$OneDimensionBKDWriter.add(BKDWriter.java:579)
app//org.apache.lucene.util.bkd.BKDWriter.merge(BKDWriter.java:504)
app//org.apache.lucene.codecs.lucene86.Lucene86PointsWriter.merge(Lucene86PointsWriter.java:237)
app//org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:201)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:161)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 11 elements
app//org.apache.lucene.util.bkd.BKDWriter$OneDimensionBKDWriter.add(BKDWriter.java:579)
app//org.apache.lucene.util.bkd.BKDWriter.merge(BKDWriter.java:504)
app//org.apache.lucene.codecs.lucene86.Lucene86PointsWriter.merge(Lucene86PointsWriter.java:237)
app//org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:201)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:161)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
3/10 snapshots sharing following 10 elements
app//org.apache.lucene.util.bkd.BKDWriter.merge(BKDWriter.java:512)
app//org.apache.lucene.codecs.lucene86.Lucene86PointsWriter.merge(Lucene86PointsWriter.java:237)
app//org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:201)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:161)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 9 elements
app//org.apache.lucene.codecs.lucene86.Lucene86PointsWriter.merge(Lucene86PointsWriter.java:237)
app//org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:201)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:161)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
4.9% [cpu=4.9%, other=0.0%] (24.5ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-5][refresh][T#2]'
4/10 snapshots sharing following 34 elements
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:891)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedSetField(PerFieldDocValuesFormat.java:129)
app//org.apache.lucene.index.SortedSetDocValuesWriter.flush(SortedSetDocValuesWriter.java:199)
app//org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:348)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:228)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
4/10 snapshots sharing following 31 elements
app//org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:348)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:228)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 34 elements
app//org.apache.lucene.util.bkd.BKDWriter.writeField(BKDWriter.java:346)
app//org.apache.lucene.codecs.lucene86.Lucene86PointsWriter.writeField(Lucene86PointsWriter.java:123)
app//org.apache.lucene.index.PointValuesWriter.flush(PointValuesWriter.java:209)
app//org.apache.lucene.index.DefaultIndexingChain.writePoints(DefaultIndexingChain.java:308)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:234)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3.0% [cpu=1.4%, other=1.6%] (15ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-5][[alerts-redacted-traffic-000052][1]: Lucene Merge Thread #228]'
10/10 snapshots sharing following 19 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.copyChunks(CompressingStoredFieldsWriter.java:583)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:633)
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:105)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2.0% [cpu=1.2%, other=0.8%] (10ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-5][[alerts-redacted-traffic-000007][0]: Lucene Merge Thread #210]'
10/10 snapshots sharing following 17 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:97)
app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5313)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4821)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
0.0% [cpu=0.0%, other=0.0%] (39.6micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'java-sdk-http-connection-reaper'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:188)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 5 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'ml-cpp-log-tail-thread'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.io.FileInputStream.readBytes(Native Method)
java.base@20.0.1/java.io.FileInputStream.read(FileInputStream.java:269)
org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:106)
org.elasticsearch.xpack.ml.process.NativeController.lambda$tailLogsInThread$0(NativeController.java:103)
org.elasticsearch.xpack.ml.process.NativeController$$Lambda$3540/0x000000080183ee58.run(Unknown Source)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'process reaper (pid 214)'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.lang.ProcessHandleImpl.waitForProcessExit0(Native Method)
java.base@20.0.1/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:163)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
java.base@20.0.1/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:186)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.17.10]'
10/10 snapshots sharing following 8 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:221)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:754)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1099)
java.base@20.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
::: {alerts-es-hot-10}{B5nmJ5MiQ_ewiN2Lph0r-A}{kkAdVazOR3SuBq-v9XncEQ}{172.16.31.35}{172.16.31.35:9300}{hir}{k8s_node_name=ip-172-16-58-144.us-east-2.compute.internal, xpack.installed=true, data=hot, transform.node=false}
Hot threads at 2023-05-23T17:43:16.187Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
100.0% [cpu=46.0%, other=54.0%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][[alerts-redacted-000099][0]: Lucene Merge Thread #516]'
2/10 snapshots sharing following 14 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:606)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:947)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
4/10 snapshots sharing following 14 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:628)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:947)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
4/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
100.0% [cpu=44.4%, other=55.6%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][[alerts-redacted-traffic-000043][1]: Lucene Merge Thread #131]'
2/10 snapshots sharing following 14 elements
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.startDoc(Lucene84PostingsWriter.java:243)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
7/10 snapshots sharing following 13 elements
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:133)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
unique snapshot
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.startDoc(Lucene84PostingsWriter.java:259)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
100.0% [cpu=44.2%, other=55.8%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][write][T#1]'
3/10 snapshots sharing following 11 elements
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
7/10 snapshots sharing following 3 elements
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=42.5%, other=57.5%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][[alerts-redacted-000036][1]: Lucene Merge Thread #404]'
2/10 snapshots sharing following 16 elements
app//org.apache.lucene.codecs.lucene84.ForUtil.encode(ForUtil.java:291)
app//org.apache.lucene.codecs.lucene84.ForDeltaUtil.encodeDeltas(ForDeltaUtil.java:67)
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.startDoc(Lucene84PostingsWriter.java:243)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
8/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
100.0% [cpu=42.1%, other=57.9%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][[alerts-redacted-traffic-000007][0]: Lucene Merge Thread #222]'
4/10 snapshots sharing following 13 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:947)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
4/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 13 elements
app//org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
app//org.apache.lucene.index.FilterLeafReader$FilterTermsEnum.next(FilterLeafReader.java:194)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:310)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
100.0% [cpu=40.0%, other=60.0%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][[alerts-redacted-traffic-000009][0]: Lucene Merge Thread #107]'
2/10 snapshots sharing following 19 elements
app//org.apache.lucene.util.packed.DirectReader$DirectPackedReader8.get(DirectReader.java:145)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$21.ordValue(Lucene80DocValuesProducer.java:988)
app//org.apache.lucene.index.SingletonSortedSetDocValues.nextDoc(SingletonSortedSetDocValues.java:66)
app//org.apache.lucene.codecs.DocValuesConsumer$SortedSetDocValuesSub.nextDoc(DocValuesConsumer.java:742)
app//org.apache.lucene.index.DocIDMerger$Sub.nextMappedDoc(DocIDMerger.java:56)
app//org.apache.lucene.index.DocIDMerger$SequentialDocIDMerger.next(DocIDMerger.java:116)
app//org.apache.lucene.codecs.DocValuesConsumer$5$1.nextDoc(DocValuesConsumer.java:848)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:882)
app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:804)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:145)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
8/10 snapshots sharing following 11 elements
app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:804)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:145)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
100.0% [cpu=38.2%, other=61.8%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][write][T#2]'
5/10 snapshots sharing following 25 elements
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$6936/0x00000008020d26e0.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 14 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue$Node.block(LinkedTransferQueue.java:470)
java.base@20.0.1/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
java.base@20.0.1/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:669)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:616)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1286)
app//org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:152)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 23 elements
app//org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:844)
app//org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:865)
app//org.elasticsearch.index.engine.InternalEngine.ensureTranslogSynced(InternalEngine.java:569)
app//org.elasticsearch.index.shard.IndexShard$7.write(IndexShard.java:3743)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.processList(AsyncIOProcessor.java:97)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.drainAndProcessAndRelease(AsyncIOProcessor.java:85)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.put(AsyncIOProcessor.java:73)
app//org.elasticsearch.index.shard.IndexShard.sync(IndexShard.java:3766)
app//org.elasticsearch.action.support.replication.TransportWriteAction$AsyncAfterWriteAction.run(TransportWriteAction.java:455)
app//org.elasticsearch.action.support.replication.TransportWriteAction$WriteReplicaResult.runPostReplicaActions(TransportWriteAction.java:336)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.lambda$onResponse$2(TransportReplicationAction.java:669)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction$$Lambda$6887/0x00000008020c69d8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:447)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=35.6%, other=64.4%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][[alerts-redacted-000030][0]: Lucene Merge Thread #947]'
10/10 snapshots sharing following 11 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
100.0% [cpu=16.7%, other=83.3%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][[associations-000905][0]: Lucene Merge Thread #217643]'
4/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 19 elements
app//org.apache.lucene.util.BytesRef.compareTo(BytesRef.java:160)
app//org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.lessThan(MultiTermsEnum.java:410)
app//org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.lessThan(MultiTermsEnum.java:399)
app//org.apache.lucene.util.PriorityQueue.downHeap(PriorityQueue.java:267)
app//org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:202)
app//org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
app//org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
app//org.apache.lucene.index.FilterLeafReader$FilterTermsEnum.next(FilterLeafReader.java:194)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:310)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
4/10 snapshots sharing following 10 elements
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
100.0% [cpu=4.1%, other=95.9%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][transport_worker][T#8]'
9/10 snapshots sharing following 4 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=3.8%, other=96.2%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][transport_worker][T#4]'
2/10 snapshots sharing following 37 elements
java.base@20.0.1/com.sun.crypto.provider.GaloisCounterMode$GCMEngine.doLastBlock(GaloisCounterMode.java:931)
java.base@20.0.1/com.sun.crypto.provider.GaloisCounterMode$GCMDecrypt.doFinal(GaloisCounterMode.java:1611)
java.base@20.0.1/com.sun.crypto.provider.GaloisCounterMode.engineDoFinal(GaloisCounterMode.java:458)
java.base@20.0.1/javax.crypto.Cipher.doFinal(Cipher.java:2543)
java.base@20.0.1/sun.security.ssl.SSLCipher$T13GcmReadCipherGenerator$GcmReadCipher.decrypt(SSLCipher.java:1908)
java.base@20.0.1/sun.security.ssl.SSLEngineInputRecord.decodeInputRecord(SSLEngineInputRecord.java:239)
java.base@20.0.1/sun.security.ssl.SSLEngineInputRecord.decode(SSLEngineInputRecord.java:196)
java.base@20.0.1/sun.security.ssl.SSLEngineInputRecord.decode(SSLEngineInputRecord.java:159)
java.base@20.0.1/sun.security.ssl.SSLTransport.decode(SSLTransport.java:111)
java.base@20.0.1/sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736)
java.base@20.0.1/sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691)
java.base@20.0.1/sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506)
java.base@20.0.1/sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482)
java.base@20.0.1/javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679)
io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:298)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1344)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1237)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1286)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
6/10 snapshots sharing following 4 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
98.0% [cpu=51.4%, other=46.6%] (490ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][write][T#7]'
2/10 snapshots sharing following 21 elements
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$6936/0x00000008020d26e0.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
5/10 snapshots sharing following 27 elements
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$6936/0x00000008020d26e0.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 15 elements
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$6936/0x00000008020d26e0.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
97.6% [cpu=37.4%, other=60.2%] (488ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][[alerts-redacted-000018][0]: Lucene Merge Thread #201]'
2/10 snapshots sharing following 15 elements
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsReader$EverythingEnum.nextPosition(Lucene84PostingsReader.java:961)
app//org.apache.lucene.index.MappingMultiPostingsEnum.nextPosition(MappingMultiPostingsEnum.java:113)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:150)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 17 elements
app//org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:212)
app//org.apache.lucene.codecs.lucene84.ForUtil.encode(ForUtil.java:291)
app//org.apache.lucene.codecs.lucene84.ForDeltaUtil.encodeDeltas(ForDeltaUtil.java:67)
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.startDoc(Lucene84PostingsWriter.java:243)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
6/10 snapshots sharing following 10 elements
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
95.4% [cpu=44.5%, other=50.9%] (477ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][write][T#3]'
5/10 snapshots sharing following 3 elements
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
95.4% [cpu=31.9%, other=63.5%] (477ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][[alerts-redacted-traffic-000052][0]: Lucene Merge Thread #224]'
8/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 16 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.append(BlockTreeTermsWriter.java:476)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.compileIndex(BlockTreeTermsWriter.java:448)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:637)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:947)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
94.0% [cpu=53.9%, other=40.1%] (470ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][write][T#4]'
6/10 snapshots sharing following 25 elements
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$6936/0x00000008020d26e0.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 22 elements
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$6936/0x00000008020d26e0.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 15 elements
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$6936/0x00000008020d26e0.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
93.6% [cpu=33.8%, other=59.8%] (468ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][write][T#5]'
7/10 snapshots sharing following 6 elements
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 14 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue$Node.block(LinkedTransferQueue.java:470)
java.base@20.0.1/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
java.base@20.0.1/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:669)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:616)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1286)
app//org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:152)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
92.4% [cpu=36.5%, other=55.9%] (462ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][write][T#6]'
2/10 snapshots sharing following 23 elements
app//org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:844)
app//org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:865)
app//org.elasticsearch.index.engine.InternalEngine.ensureTranslogSynced(InternalEngine.java:569)
app//org.elasticsearch.index.shard.IndexShard$7.write(IndexShard.java:3743)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.processList(AsyncIOProcessor.java:97)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.drainAndProcessAndRelease(AsyncIOProcessor.java:85)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.put(AsyncIOProcessor.java:73)
app//org.elasticsearch.index.shard.IndexShard.sync(IndexShard.java:3766)
app//org.elasticsearch.action.support.replication.TransportWriteAction$AsyncAfterWriteAction.run(TransportWriteAction.java:455)
app//org.elasticsearch.action.support.replication.TransportWriteAction$WriteReplicaResult.runPostReplicaActions(TransportWriteAction.java:336)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.lambda$onResponse$2(TransportReplicationAction.java:669)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction$$Lambda$6887/0x00000008020c69d8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:447)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
7/10 snapshots sharing following 14 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue$Node.block(LinkedTransferQueue.java:470)
java.base@20.0.1/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
java.base@20.0.1/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:669)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:616)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1286)
app//org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:152)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
java.base@20.0.1/java.util.HashMap.putVal(HashMap.java:669)
java.base@20.0.1/java.util.HashMap.put(HashMap.java:618)
app//org.elasticsearch.common.time.EpochTime$3.resolve(EpochTime.java:101)
java.base@20.0.1/java.time.format.Parsed.resolveFields(Parsed.java:289)
java.base@20.0.1/java.time.format.Parsed.resolve(Parsed.java:265)
java.base@20.0.1/java.time.format.DateTimeParseContext.toResolved(DateTimeParseContext.java:331)
java.base@20.0.1/java.time.format.DateTimeFormatter$ClassicFormat.parseObject(DateTimeFormatter.java:2310)
app//org.elasticsearch.common.time.JavaDateFormatter.doParse(JavaDateFormatter.java:233)
app//org.elasticsearch.common.time.JavaDateFormatter.parse(JavaDateFormatter.java:209)
app//org.elasticsearch.index.mapper.DateFieldMapper$DateFieldType.parse(DateFieldMapper.java:432)
app//org.elasticsearch.index.mapper.DateFieldMapper.parseCreateField(DateFieldMapper.java:766)
app//org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:229)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:577)
app//org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:741)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$6936/0x00000008020d26e0.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
92.2% [cpu=31.7%, other=60.5%] (461ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][write][T#8]'
2/10 snapshots sharing following 30 elements
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:260)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:172)
app//org.apache.lucene.index.StoredFieldsConsumer.finishDocument(StoredFieldsConsumer.java:68)
app//org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:463)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:499)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$6936/0x00000008020d26e0.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
5/10 snapshots sharing following 17 elements
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$6936/0x00000008020d26e0.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 29 elements
app//org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:259)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:577)
app//org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:741)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:574)
app//org.elasticsearch.index.mapper.DocumentParser.parseObject(DocumentParser.java:608)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:502)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$6936/0x00000008020d26e0.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
app//com.fasterxml.jackson.core.base.ParserBase.close(ParserBase.java:390)
app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextToken(UTF8StreamJsonParser.java:759)
app//org.elasticsearch.xcontent.json.JsonXContentParser.nextToken(JsonXContentParser.java:73)
app//org.elasticsearch.index.mapper.DocumentParser.validateEnd(DocumentParser.java:212)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:92)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$6936/0x00000008020d26e0.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
43.6% [cpu=14.9%, other=28.7%] (218ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][refresh][T#2]'
2/10 snapshots sharing following 36 elements
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.startDoc(Lucene84PostingsWriter.java:259)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:120)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:267)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
7/10 snapshots sharing following 33 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:120)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:267)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.encodeTerm(Lucene84PostingsWriter.java:475)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlock(BlockTreeTermsWriter.java:735)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:606)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.finish(BlockTreeTermsWriter.java:976)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:321)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:120)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:267)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
36.6% [cpu=17.2%, other=19.4%] (183ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][refresh][T#1]'
2/10 snapshots sharing following 35 elements
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:120)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:267)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 39 elements
app//org.apache.lucene.util.fst.BytesStore.writeByte(BytesStore.java:90)
app//org.apache.lucene.util.fst.FST.<init>(FST.java:404)
app//org.apache.lucene.util.fst.Builder.<init>(Builder.java:189)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.compileIndex(BlockTreeTermsWriter.java:431)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:637)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.finish(BlockTreeTermsWriter.java:976)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:321)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:120)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:267)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 28 elements
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 10 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:676)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:616)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1294)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
1.7% [cpu=1.7%, other=0.0%] (8.6ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][[alerts-redacted-traffic-000012][0]: Lucene Merge Thread #290]'
10/10 snapshots sharing following 17 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:97)
app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5313)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4821)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
1.7% [cpu=1.7%, other=0.0%] (8.3ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][[alerts-redacted-traffic-000206][2]: Lucene Merge Thread #103]'
10/10 snapshots sharing following 17 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:97)
app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5313)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4821)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
1.5% [cpu=1.5%, other=0.0%] (7.5ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][[alerts-redacted-traffic-000011][0]: Lucene Merge Thread #285]'
10/10 snapshots sharing following 17 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:97)
app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5313)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4821)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
1.3% [cpu=1.3%, other=0.0%] (6.2ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-10][[alerts-redacted-traffic-000007][0]: Lucene Merge Thread #313]'
7/10 snapshots sharing following 17 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:97)
app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5313)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4821)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 24 elements
java.base@20.0.1/sun.nio.ch.UnixFileDispatcherImpl.write0(Native Method)
java.base@20.0.1/sun.nio.ch.UnixFileDispatcherImpl.write(UnixFileDispatcherImpl.java:65)
java.base@20.0.1/sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:137)
java.base@20.0.1/sun.nio.ch.IOUtil.write(IOUtil.java:102)
java.base@20.0.1/sun.nio.ch.IOUtil.write(IOUtil.java:72)
java.base@20.0.1/sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:300)
java.base@20.0.1/sun.nio.ch.ChannelOutputStream.writeFully(ChannelOutputStream.java:68)
java.base@20.0.1/sun.nio.ch.ChannelOutputStream.write(ChannelOutputStream.java:105)
app//org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:416)
java.base@20.0.1/java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73)
java.base@20.0.1/java.io.BufferedOutputStream.implWrite(BufferedOutputStream.java:217)
java.base@20.0.1/java.io.BufferedOutputStream.write(BufferedOutputStream.java:200)
app//org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53)
app//org.elasticsearch.common.lucene.store.FilterIndexOutput.writeBytes(FilterIndexOutput.java:48)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:73)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:97)
app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5313)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4821)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
unique snapshot
java.base@20.0.1/sun.nio.fs.UnixNativeDispatcher.unlink0(Native Method)
java.base@20.0.1/sun.nio.fs.UnixNativeDispatcher.unlink(UnixNativeDispatcher.java:159)
java.base@20.0.1/sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:278)
java.base@20.0.1/sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:104)
java.base@20.0.1/java.nio.file.Files.delete(Files.java:1151)
app//org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:370)
app//org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:339)
app//org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63)
app//org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:170)
app//org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63)
app//org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:745)
app//org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:750)
app//org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38)
app//org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:705)
app//org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:699)
app//org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:673)
app//org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:5333)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4856)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
0.0% [cpu=0.0%, other=0.0%] (36.1micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'java-sdk-http-connection-reaper'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:188)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.17.10]'
10/10 snapshots sharing following 8 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:221)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:754)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1099)
java.base@20.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'process reaper (pid 213)'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.lang.ProcessHandleImpl.waitForProcessExit0(Native Method)
java.base@20.0.1/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:163)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
java.base@20.0.1/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:186)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'ml-cpp-log-tail-thread'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.io.FileInputStream.readBytes(Native Method)
java.base@20.0.1/java.io.FileInputStream.read(FileInputStream.java:269)
org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:106)
org.elasticsearch.xpack.ml.process.NativeController.lambda$tailLogsInThread$0(NativeController.java:103)
org.elasticsearch.xpack.ml.process.NativeController$$Lambda$3522/0x000000080183ae58.run(Unknown Source)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 5 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
::: {alerts-es-hot-6}{VT5bS47MSZu6SJRk_7eURw}{3rH4648FRxyLk8sA-O4jSA}{172.16.47.205}{172.16.47.205:9300}{hir}{k8s_node_name=ip-172-16-3-136.us-east-2.compute.internal, xpack.installed=true, data=hot, transform.node=false}
Hot threads at 2023-05-23T17:43:16.285Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
100.0% [cpu=89.5%, other=10.5%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-6][write][T#1]'
3/10 snapshots sharing following 30 elements
app//org.apache.lucene.index.TermsHashPerField.start(TermsHashPerField.java:313)
app//org.apache.lucene.index.FreqProxTermsWriterPerField.start(FreqProxTermsWriterPerField.java:66)
app//org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:922)
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7199/0x0000000802090000.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
6/10 snapshots sharing following 25 elements
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7199/0x0000000802090000.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:151)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7199/0x0000000802090000.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=81.1%, other=18.9%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-6][write][T#5]'
2/10 snapshots sharing following 30 elements
app//org.apache.lucene.analysis.standard.StandardTokenizer.incrementToken(StandardTokenizer.java:148)
app//org.apache.lucene.analysis.LowerCaseFilter.incrementToken(LowerCaseFilter.java:41)
app//org.apache.lucene.analysis.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:49)
app//org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:924)
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
5/10 snapshots sharing following 24 elements
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 20 elements
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
java.base@20.0.1/java.lang.ref.Reference.clear0(Native Method)
java.base@20.0.1/java.lang.ref.Reference.clear(Reference.java:398)
java.base@20.0.1/java.lang.ThreadLocal$ThreadLocalMap.remove(ThreadLocal.java:587)
java.base@20.0.1/java.lang.ThreadLocal.remove(ThreadLocal.java:298)
java.base@20.0.1/java.lang.ThreadLocal.remove(ThreadLocal.java:287)
java.base@20.0.1/java.util.concurrent.locks.ReentrantReadWriteLock$Sync.tryReleaseShared(ReentrantReadWriteLock.java:430)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.releaseShared(AbstractQueuedSynchronizer.java:1146)
java.base@20.0.1/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.unlock(ReentrantReadWriteLock.java:897)
app//org.elasticsearch.common.util.concurrent.ReleasableLock.close(ReleasableLock.java:38)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1099)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=79.3%, other=20.7%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-6][write][T#2]'
3/10 snapshots sharing following 27 elements
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7199/0x0000000802090000.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 26 elements
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7199/0x0000000802090000.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 17 elements
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7199/0x0000000802090000.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 20 elements
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:151)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7199/0x0000000802090000.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
java.base@20.0.1/java.util.HashMap.putMapEntries(HashMap.java:503)
java.base@20.0.1/java.util.HashMap.putAll(HashMap.java:791)
java.base@20.0.1/java.time.format.Parsed.copy(Parsed.java:181)
java.base@20.0.1/java.time.format.DateTimeParseContext.startOptional(DateTimeParseContext.java:287)
java.base@20.0.1/java.time.format.DateTimeFormatterBuilder$CompositePrinterParser.parse(DateTimeFormatterBuilder.java:2541)
java.base@20.0.1/java.time.format.DateTimeFormatterBuilder$CompositePrinterParser.parse(DateTimeFormatterBuilder.java:2554)
java.base@20.0.1/java.time.format.DateTimeFormatter.parseUnresolved0(DateTimeFormatter.java:2167)
java.base@20.0.1/java.time.format.DateTimeFormatter$ClassicFormat.parseObject(DateTimeFormatter.java:2296)
app//org.elasticsearch.common.time.JavaDateFormatter.doParse(JavaDateFormatter.java:233)
app//org.elasticsearch.common.time.JavaDateFormatter.parse(JavaDateFormatter.java:209)
app//org.elasticsearch.index.mapper.DateFieldMapper$DateFieldType.parse(DateFieldMapper.java:432)
app//org.elasticsearch.index.mapper.DateFieldMapper.parseCreateField(DateFieldMapper.java:766)
app//org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:229)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:577)
app//org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:741)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7199/0x0000000802090000.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=68.0%, other=32.0%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-6][write][T#6]'
4/10 snapshots sharing following 27 elements
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7199/0x0000000802090000.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 26 elements
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7199/0x0000000802090000.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 21 elements
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7199/0x0000000802090000.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
java.base@20.0.1/java.util.Objects.equals(Objects.java:64)
app//org.elasticsearch.index.mapper.DocumentParser.validateType(DocumentParser.java:160)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:77)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7199/0x0000000802090000.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=60.8%, other=39.2%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-6][write][T#8]'
2/10 snapshots sharing following 21 elements
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7199/0x0000000802090000.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 27 elements
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7199/0x0000000802090000.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 9 elements
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 26 elements
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.getText(UTF8StreamJsonParser.java:334)
app//org.elasticsearch.xcontent.json.JsonXContentParser.text(JsonXContentParser.java:104)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.textOrNull(AbstractXContentParser.java:241)
app//org.elasticsearch.index.mapper.IpFieldMapper.value(IpFieldMapper.java:488)
app//org.elasticsearch.index.mapper.IpFieldMapper.parseCreateField(IpFieldMapper.java:473)
app//org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:229)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:577)
app//org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:741)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=4.8%, other=95.2%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-6][transport_worker][T#7]'
6/10 snapshots sharing following 4 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
99.6% [cpu=58.0%, other=41.6%] (498ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-6][write][T#7]'
10/10 snapshots sharing following 14 elements
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
97.8% [cpu=74.6%, other=23.2%] (489ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-6][write][T#3]'
4/10 snapshots sharing following 26 elements
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 26 elements
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:564)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 16 elements
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 11 elements
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
96.6% [cpu=51.7%, other=44.9%] (483ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-6][[alerts-redacted-traffic-000007][0]: Lucene Merge Thread #264]'
2/10 snapshots sharing following 12 elements
app//org.apache.lucene.index.FilterLeafReader$FilterTermsEnum.next(FilterLeafReader.java:194)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:310)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
3/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
5/10 snapshots sharing following 13 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:947)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
96.0% [cpu=58.6%, other=37.4%] (480ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-6][write][T#4]'
4/10 snapshots sharing following 26 elements
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 20 elements
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 25 elements
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 14 elements
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
89.2% [cpu=66.2%, other=23.0%] (446ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-6][[alerts-redacted-traffic-000052][1]: Lucene Merge Thread #254]'
8/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 17 elements
app//org.apache.lucene.util.compress.LZ4.compressWithDictionary(LZ4.java:470)
app//org.apache.lucene.util.compress.LZ4.compress(LZ4.java:419)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlock(BlockTreeTermsWriter.java:823)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:606)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:947)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
79.4% [cpu=51.2%, other=28.2%] (397ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-6][[alerts-redacted-traffic-000012][0]: Lucene Merge Thread #209]'
4/10 snapshots sharing following 16 elements
app//org.apache.lucene.codecs.lucene84.ForUtil.encode(ForUtil.java:291)
app//org.apache.lucene.codecs.lucene84.PForUtil.encode(PForUtil.java:93)
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.addPosition(Lucene84PostingsWriter.java:310)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:161)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 14 elements
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.startDoc(Lucene84PostingsWriter.java:245)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 10 elements
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2.8% [cpu=2.4%, other=0.4%] (14ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-6][[alerts-redacted-traffic-000017][0]: Lucene Merge Thread #16]'
10/10 snapshots sharing following 19 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.copyChunks(CompressingStoredFieldsWriter.java:583)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:633)
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:105)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
0.1% [cpu=0.1%, other=0.0%] (393.4micros out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-6][management][T#1]'
5/10 snapshots sharing following 58 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsReader.ramBytesUsed(BlockTreeTermsReader.java:332)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.ramBytesUsed(PerFieldPostingsFormat.java:358)
app//org.apache.lucene.index.CodecReader.ramBytesUsed(CodecReader.java:214)
app//org.elasticsearch.index.engine.Engine.fillSegmentStats(Engine.java:874)
app//org.elasticsearch.index.engine.Engine.segmentsStats(Engine.java:856)
app//org.elasticsearch.index.shard.IndexShard.segmentStats(IndexShard.java:1387)
app//org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:207)
app//org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:520)
app//org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:474)
app//org.elasticsearch.indices.IndicesService.stats(IndicesService.java:465)
app//org.elasticsearch.node.NodeService.stats(NodeService.java:149)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:87)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:80)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:32)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:333)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:327)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:341)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:404)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:394)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$4(AuthorizationService.java:397)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$6295/0x0000000801efc6c0.accept(Unknown Source)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:964)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:928)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$5(AuthorizationService.java:411)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$6297/0x0000000801efcb10.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authz.RBACEngine.authorizeClusterAction(RBACEngine.java:178)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:401)
org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:377)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$1(AuthorizationService.java:262)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$6267/0x0000000801eec268.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$1(RBACEngine.java:148)
org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$6269/0x0000000801eec6a8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:277)
org.elasticsearch.xpack.security.authz.RBACEngine.getRoles(RBACEngine.java:154)
org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:137)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:264)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:136)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$6170/0x0000000801d834a8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:102)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:199)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:128)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:415)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:67)
app//org.elasticsearch.transport.InboundHandler$1.doRun(InboundHandler.java:272)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (49.6micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'java-sdk-http-connection-reaper'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:188)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 5 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'ml-cpp-log-tail-thread'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.io.FileInputStream.readBytes(Native Method)
java.base@20.0.1/java.io.FileInputStream.read(FileInputStream.java:269)
org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:106)
org.elasticsearch.xpack.ml.process.NativeController.lambda$tailLogsInThread$0(NativeController.java:103)
org.elasticsearch.xpack.ml.process.NativeController$$Lambda$3540/0x0000000801883cd0.run(Unknown Source)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'process reaper (pid 214)'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.lang.ProcessHandleImpl.waitForProcessExit0(Native Method)
java.base@20.0.1/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:163)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
java.base@20.0.1/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:186)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.17.10]'
10/10 snapshots sharing following 8 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:221)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:754)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1099)
java.base@20.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
::: {alerts-es-hot-7}{OT3gVh17SxCOPkJwTcOoRg}{VL_MgLGBQ-uq4HcBdafqFg}{172.16.3.211}{172.16.3.211:9300}{hir}{k8s_node_name=ip-172-16-41-38.us-east-2.compute.internal, xpack.installed=true, data=hot, transform.node=false}
Hot threads at 2023-05-23T17:43:16.187Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
100.0% [cpu=3.3%, other=96.7%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-7][transport_worker][T#4]'
unique snapshot
org.elasticsearch.xpack.core.security.user.InternalUserSerializationHelper.readFrom(InternalUserSerializationHelper.java:30)
org.elasticsearch.xpack.core.security.authc.Authentication.<init>(Authentication.java:72)
org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer.decode(AuthenticationContextSerializer.java:64)
org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer.deserializeHeaderAndPutInContext(AuthenticationContextSerializer.java:54)
org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer.readFromContext(AuthenticationContextSerializer.java:48)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lookForExistingAuthentication(AuthenticatorChain.java:244)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:95)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:199)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:128)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:415)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:67)
app//org.elasticsearch.transport.InboundHandler.handleRequest(InboundHandler.java:261)
app//org.elasticsearch.transport.InboundHandler.messageReceived(InboundHandler.java:109)
app//org.elasticsearch.transport.InboundHandler.inboundMessage(InboundHandler.java:88)
app//org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:743)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$6040/0x0000000801e9a780.accept(Unknown Source)
app//org.elasticsearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:147)
app//org.elasticsearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:119)
app//org.elasticsearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:84)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1374)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1237)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1286)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
79.9% [cpu=79.9%, other=0.0%] (399.5ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-7][[alerts-redacted-traffic-000007][0]: Lucene Merge Thread #448]'
2/10 snapshots sharing following 17 elements
app//org.apache.lucene.index.DocIDMerger$Sub.nextMappedDoc(DocIDMerger.java:56)
app//org.apache.lucene.index.DocIDMerger$SequentialDocIDMerger.next(DocIDMerger.java:116)
app//org.apache.lucene.codecs.DocValuesConsumer$5$1.nextDoc(DocValuesConsumer.java:848)
app//org.apache.lucene.search.SortedSetSelector$MinValue.nextDoc(SortedSetSelector.java:108)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.doAddSortedField(Lucene80DocValuesConsumer.java:675)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:891)
app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:804)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:145)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
7/10 snapshots sharing following 16 elements
app//org.apache.lucene.store.DataOutput.writeBytes(DataOutput.java:52)
app//org.apache.lucene.util.packed.DirectWriter.flush(DirectWriter.java:86)
app//org.apache.lucene.util.packed.DirectWriter.add(DirectWriter.java:78)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.doAddSortedField(Lucene80DocValuesConsumer.java:676)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:891)
app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:804)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:145)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
unique snapshot
app//org.apache.lucene.util.packed.BulkOperationPacked.encode(BulkOperationPacked.java:222)
app//org.apache.lucene.util.packed.DirectWriter.flush(DirectWriter.java:84)
app//org.apache.lucene.util.packed.DirectWriter.add(DirectWriter.java:78)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.doAddSortedField(Lucene80DocValuesConsumer.java:676)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:891)
app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:804)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:145)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
40.6% [cpu=40.6%, other=0.0%] (203.1ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-7][[alerts-redacted-traffic-000007][0]: Lucene Merge Thread #278]'
2/10 snapshots sharing following 14 elements
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.startDoc(Lucene84PostingsWriter.java:243)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
8/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
31.7% [cpu=31.7%, other=0.0%] (158.4ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-7][write][T#1]'
7/10 snapshots sharing following 17 elements
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7220/0x00000008020d8240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 15 elements
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7220/0x00000008020d8240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:538)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7220/0x00000008020d8240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
25.2% [cpu=25.2%, other=0.0%] (125.8ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-7][write][T#4]'
2/10 snapshots sharing following 30 elements
app//org.apache.lucene.analysis.LowerCaseFilter.incrementToken(LowerCaseFilter.java:41)
app//org.apache.lucene.analysis.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:49)
app//org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:924)
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7220/0x00000008020d8240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
8/10 snapshots sharing following 3 elements
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
19.1% [cpu=19.1%, other=0.0%] (95.3ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-7][write][T#8]'
3/10 snapshots sharing following 25 elements
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7220/0x00000008020d8240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 9 elements
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
4/10 snapshots sharing following 14 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue$Node.block(LinkedTransferQueue.java:470)
java.base@20.0.1/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
java.base@20.0.1/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:669)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:616)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1286)
app//org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:152)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
15.6% [cpu=15.4%, other=0.2%] (78ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-7][write][T#5]'
6/10 snapshots sharing following 15 elements
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7220/0x00000008020d8240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
11.6% [cpu=11.6%, other=0.0%] (57.8ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-7][write][T#7]'
3/10 snapshots sharing following 26 elements
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7220/0x00000008020d8240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
4/10 snapshots sharing following 21 elements
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7220/0x00000008020d8240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 18 elements
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7220/0x00000008020d8240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
app//org.apache.lucene.util.compress.LZ4.compressWithDictionary(LZ4.java:479)
app//org.apache.lucene.codecs.lucene87.LZ4WithPresetDictCompressionMode$LZ4WithPresetDictCompressor.doCompress(LZ4WithPresetDictCompressionMode.java:162)
app//org.apache.lucene.codecs.lucene87.LZ4WithPresetDictCompressionMode$LZ4WithPresetDictCompressor.compress(LZ4WithPresetDictCompressionMode.java:185)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:260)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:172)
app//org.apache.lucene.index.StoredFieldsConsumer.finishDocument(StoredFieldsConsumer.java:68)
app//org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:463)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:499)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7220/0x00000008020d8240.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2.8% [cpu=2.8%, other=0.0%] (14.2ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-7][write][T#2]'
2/10 snapshots sharing following 28 elements
app//org.apache.lucene.document.Field.tokenStream(Field.java:513)
app//org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:918)
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
6/10 snapshots sharing following 25 elements
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
app//org.apache.lucene.search.similarities.PerFieldSimilarityWrapper.computeNorm(PerFieldSimilarityWrapper.java:42)
app//org.apache.lucene.index.DefaultIndexingChain$PerField.finish(DefaultIndexingChain.java:880)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:497)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
1.4% [cpu=1.4%, other=0.0%] (6.9ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-7][[associations-000905][0]: Lucene Merge Thread #193843]'
10/10 snapshots sharing following 17 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:97)
app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5313)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4821)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
1.1% [cpu=1.1%, other=0.0%] (5.3ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-7][[alerts-redacted-traffic-000007][0]: Lucene Merge Thread #340]'
10/10 snapshots sharing following 19 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.copyChunks(CompressingStoredFieldsWriter.java:583)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:633)
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:105)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
0.0% [cpu=0.0%, other=0.0%] (49.7micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'java-sdk-http-connection-reaper'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:188)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 5 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'ml-cpp-log-tail-thread'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.io.FileInputStream.readBytes(Native Method)
java.base@20.0.1/java.io.FileInputStream.read(FileInputStream.java:269)
org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:106)
org.elasticsearch.xpack.ml.process.NativeController.lambda$tailLogsInThread$0(NativeController.java:103)
org.elasticsearch.xpack.ml.process.NativeController$$Lambda$3540/0x000000080183ee58.run(Unknown Source)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'process reaper (pid 214)'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.lang.ProcessHandleImpl.waitForProcessExit0(Native Method)
java.base@20.0.1/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:163)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
java.base@20.0.1/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:186)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.17.10]'
10/10 snapshots sharing following 8 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:221)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:754)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1099)
java.base@20.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
::: {alerts-es-hot-0}{to8NRjbsQbSqU0JBUDs-ZA}{aoEOd9XoTxOzL8NTKVpSdg}{172.16.54.28}{172.16.54.28:9300}{hir}{k8s_node_name=ip-172-16-26-231.us-east-2.compute.internal, xpack.installed=true, data=hot, transform.node=false}
Hot threads at 2023-05-23T17:43:16.187Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
100.0% [cpu=5.0%, other=95.0%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-0][transport_worker][T#8]'
10/10 snapshots sharing following 4 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
97.1% [cpu=97.1%, other=0.0%] (485.4ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-0][write][T#4]'
3/10 snapshots sharing following 14 elements
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
7/10 snapshots sharing following 14 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue$Node.block(LinkedTransferQueue.java:470)
java.base@20.0.1/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
java.base@20.0.1/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:669)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:616)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1286)
app//org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:152)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
33.7% [cpu=33.7%, other=0.0%] (168.2ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-0][[alerts-redacted-traffic-000009][0]: Lucene Merge Thread #235]'
7/10 snapshots sharing following 13 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:947)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
3/10 snapshots sharing following 11 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
23.2% [cpu=23.2%, other=0.0%] (116.2ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-0][write][T#2]'
2/10 snapshots sharing following 31 elements
app//org.apache.lucene.analysis.standard.StandardTokenizerImpl.getNextToken(StandardTokenizerImpl.java:924)
app//org.apache.lucene.analysis.standard.StandardTokenizer.incrementToken(StandardTokenizer.java:148)
app//org.apache.lucene.analysis.LowerCaseFilter.incrementToken(LowerCaseFilter.java:41)
app//org.apache.lucene.analysis.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:49)
app//org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:924)
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
4/10 snapshots sharing following 25 elements
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 21 elements
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 15 elements
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
8.0% [cpu=8.0%, other=0.0%] (40ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-0][write][T#6]'
5/10 snapshots sharing following 26 elements
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7400/0x000000080210e8b8.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 21 elements
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7400/0x000000080210e8b8.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 17 elements
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7400/0x000000080210e8b8.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
app//org.elasticsearch.index.shard.IndexingOperationListener$CompositeListener.postIndex(IndexingOperationListener.java:95)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1102)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7400/0x000000080210e8b8.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
1.1% [cpu=1.1%, other=0.0%] (5.5ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-0][[alerts-redacted-traffic-000206][1]: Lucene Merge Thread #100]'
10/10 snapshots sharing following 17 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:97)
app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5313)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4821)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
0.0% [cpu=0.0%, other=0.0%] (48.5micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'java-sdk-http-connection-reaper'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:188)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 5 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'ml-cpp-log-tail-thread'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.io.FileInputStream.readBytes(Native Method)
java.base@20.0.1/java.io.FileInputStream.read(FileInputStream.java:269)
org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:106)
org.elasticsearch.xpack.ml.process.NativeController.lambda$tailLogsInThread$0(NativeController.java:103)
org.elasticsearch.xpack.ml.process.NativeController$$Lambda$3538/0x000000080185fcd0.run(Unknown Source)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'process reaper (pid 214)'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.lang.ProcessHandleImpl.waitForProcessExit0(Native Method)
java.base@20.0.1/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:163)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
java.base@20.0.1/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:186)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.17.10]'
10/10 snapshots sharing following 8 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:221)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:754)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1099)
java.base@20.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
::: {alerts-es-hot-1}{-oifKSGuQ6qAMhkqkWmgEA}{Ocx8PbOQRPSTTGP_8iuPCQ}{172.16.21.17}{172.16.21.17:9300}{hir}{k8s_node_name=ip-172-16-56-48.us-east-2.compute.internal, xpack.installed=true, data=hot, transform.node=false}
Hot threads at 2023-05-23T17:43:16.187Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
38.8% [cpu=38.8%, other=0.0%] (194.1ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-1][[alerts-redacted-traffic-000007][0]: Lucene Merge Thread #306]'
2/10 snapshots sharing following 26 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeByte(RateLimitedIndexOutput.java:65)
app//org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:71)
app//org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:212)
app//org.apache.lucene.codecs.lucene84.ForUtil.encode(ForUtil.java:291)
app//org.apache.lucene.codecs.lucene84.PForUtil.encode(PForUtil.java:93)
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.addPosition(Lucene84PostingsWriter.java:310)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:161)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
3/10 snapshots sharing following 18 elements
app//org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:73)
app//org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:213)
app//org.apache.lucene.codecs.lucene84.ForUtil.encode(ForUtil.java:291)
app//org.apache.lucene.codecs.lucene84.ForDeltaUtil.encodeDeltas(ForDeltaUtil.java:67)
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.startDoc(Lucene84PostingsWriter.java:243)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 13 elements
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:161)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 14 elements
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.startDoc(Lucene84PostingsWriter.java:266)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
unique snapshot
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeByte(RateLimitedIndexOutput.java:65)
app//org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:72)
app//org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:212)
app//org.apache.lucene.codecs.lucene84.ForUtil.encode(ForUtil.java:291)
app//org.apache.lucene.codecs.lucene84.ForDeltaUtil.encodeDeltas(ForDeltaUtil.java:67)
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.startDoc(Lucene84PostingsWriter.java:243)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
9.3% [cpu=9.3%, other=0.0%] (46.6ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-1][write][T#5]'
4/10 snapshots sharing following 27 elements
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7268/0x00000008020cef50.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 22 elements
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:518)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7268/0x00000008020cef50.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 25 elements
app//org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:229)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:577)
app//org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:741)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7268/0x00000008020cef50.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 16 elements
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7268/0x00000008020cef50.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
1.0% [cpu=1.0%, other=0.0%] (5.1ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-1][[alerts-redacted-traffic-000009][0]: Lucene Merge Thread #65]'
10/10 snapshots sharing following 19 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.copyChunks(CompressingStoredFieldsWriter.java:583)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:633)
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:105)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
0.9% [cpu=0.9%, other=0.0%] (4.4ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-1][[alerts-redacted-traffic-000007][0]: Lucene Merge Thread #304]'
10/10 snapshots sharing following 19 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.copyChunks(CompressingStoredFieldsWriter.java:583)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:633)
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:105)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
0.0% [cpu=0.0%, other=0.0%] (32.3micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'java-sdk-http-connection-reaper'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:188)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 5 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'ml-cpp-log-tail-thread'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.io.FileInputStream.readBytes(Native Method)
java.base@20.0.1/java.io.FileInputStream.read(FileInputStream.java:269)
org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:106)
org.elasticsearch.xpack.ml.process.NativeController.lambda$tailLogsInThread$0(NativeController.java:103)
org.elasticsearch.xpack.ml.process.NativeController$$Lambda$3538/0x000000080188bcd0.run(Unknown Source)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'process reaper (pid 214)'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.lang.ProcessHandleImpl.waitForProcessExit0(Native Method)
java.base@20.0.1/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:163)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
java.base@20.0.1/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:186)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.17.10]'
10/10 snapshots sharing following 8 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:221)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:754)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1099)
java.base@20.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
::: {alerts-es-hot-3}{fUdxelfSTXKizkokaZTwow}{pTlbz0j8SKSy7XtXajbsQw}{172.16.45.238}{172.16.45.238:9300}{hir}{k8s_node_name=ip-172-16-60-95.us-east-2.compute.internal, xpack.installed=true, data=hot, transform.node=false}
Hot threads at 2023-05-23T17:43:16.187Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
14.8% [cpu=14.8%, other=0.0%] (73.9ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-3][[alerts-redacted-traffic-000206][1]: Lucene Merge Thread #89]'
8/10 snapshots sharing following 23 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.writeBytes(DataOutput.java:52)
app//org.apache.lucene.util.packed.DirectWriter.flush(DirectWriter.java:86)
app//org.apache.lucene.util.packed.DirectWriter.add(DirectWriter.java:78)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:925)
app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:804)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:145)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:924)
app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:804)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:145)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
10.4% [cpu=10.4%, other=0.0%] (51.7ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-3][write][T#8]'
2/10 snapshots sharing following 30 elements
app//org.apache.lucene.analysis.standard.StandardTokenizer.incrementToken(StandardTokenizer.java:148)
app//org.apache.lucene.analysis.LowerCaseFilter.incrementToken(LowerCaseFilter.java:41)
app//org.apache.lucene.analysis.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:49)
app//org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:924)
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
4/10 snapshots sharing following 25 elements
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 23 elements
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:577)
app//org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:741)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
app//org.elasticsearch.index.mapper.DocumentParser.getMapper(DocumentParser.java:935)
app//org.elasticsearch.index.mapper.DocumentParser.getLeafMapper(DocumentParser.java:971)
app//org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:739)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2.9% [cpu=2.9%, other=0.0%] (14.5ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-3][[alerts-redacted-traffic-000009][0]: Lucene Merge Thread #6]'
10/10 snapshots sharing following 19 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.copyChunks(CompressingStoredFieldsWriter.java:583)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:633)
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:105)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
1.1% [cpu=1.1%, other=0.0%] (5.4ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-3][[alerts-redacted-000010][0]: Lucene Merge Thread #804]'
10/10 snapshots sharing following 17 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:97)
app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5313)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4821)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
0.0% [cpu=0.0%, other=0.0%] (107.8micros out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-3][refresh][T#3]'
2/10 snapshots sharing following 25 elements
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 10 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:676)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:616)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1294)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (24.7micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'java-sdk-http-connection-reaper'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:188)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-3][write][T#6]'
unique snapshot
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:525)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7272/0x00000008020ba250.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.17.10]'
10/10 snapshots sharing following 8 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:221)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:754)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1099)
java.base@20.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'process reaper (pid 214)'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.lang.ProcessHandleImpl.waitForProcessExit0(Native Method)
java.base@20.0.1/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:163)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
java.base@20.0.1/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:186)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 5 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'ml-cpp-log-tail-thread'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.io.FileInputStream.readBytes(Native Method)
java.base@20.0.1/java.io.FileInputStream.read(FileInputStream.java:269)
org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:106)
org.elasticsearch.xpack.ml.process.NativeController.lambda$tailLogsInThread$0(NativeController.java:103)
org.elasticsearch.xpack.ml.process.NativeController$$Lambda$3538/0x000000080188ae58.run(Unknown Source)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
::: {alerts-es-hot-9}{I7w2WbH1QGmN71-6r6h49A}{YM_-emvMQvOcnxyWjk9KwQ}{172.16.17.165}{172.16.17.165:9300}{hir}{k8s_node_name=ip-172-16-23-158.us-east-2.compute.internal, xpack.installed=true, data=hot, transform.node=false}
Hot threads at 2023-05-23T17:43:16.187Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
100.0% [cpu=9.5%, other=90.5%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-9][transport_worker][T#8]'
2/10 snapshots sharing following 68 elements
org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$6780/0x0000000801f4ed48.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListenerDirectly(ListenableFuture.java:113)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:55)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:41)
org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.getAsync(AuthorizationService.java:1012)
org.elasticsearch.xpack.security.authz.RBACEngine.authorizeIndexAction(RBACEngine.java:380)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:440)
org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:377)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$1(AuthorizationService.java:262)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$6233/0x0000000801e72610.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$1(RBACEngine.java:148)
org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$6235/0x0000000801e72a50.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.roles(CompositeRolesStore.java:173)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:279)
org.elasticsearch.xpack.security.authz.RBACEngine.getRoles(RBACEngine.java:154)
org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:137)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:264)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:136)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$6144/0x0000000801e718b8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:102)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:199)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:128)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:415)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:67)
app//org.elasticsearch.transport.InboundHandler.handleRequest(InboundHandler.java:261)
app//org.elasticsearch.transport.InboundHandler.messageReceived(InboundHandler.java:109)
app//org.elasticsearch.transport.InboundHandler.inboundMessage(InboundHandler.java:88)
app//org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:743)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler$$Lambda$6022/0x0000000801e84a50.accept(Unknown Source)
app//org.elasticsearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:147)
app//org.elasticsearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:119)
app//org.elasticsearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:84)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:71)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1374)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1237)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1286)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 10 elements
java.base@20.0.1/sun.nio.ch.EPoll.wait(Native Method)
java.base@20.0.1/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
java.base@20.0.1/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
java.base@20.0.1/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:810)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=6.8%, other=93.2%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-9][transport_worker][T#6]'
7/10 snapshots sharing following 4 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=3.9%, other=96.1%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-9][transport_worker][T#2]'
2/10 snapshots sharing following 16 elements
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
6/10 snapshots sharing following 10 elements
java.base@20.0.1/sun.nio.ch.EPoll.wait(Native Method)
java.base@20.0.1/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
java.base@20.0.1/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
java.base@20.0.1/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:810)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
java.base@20.0.1/sun.nio.ch.SocketDispatcher.write0(Native Method)
java.base@20.0.1/sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62)
java.base@20.0.1/sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:137)
java.base@20.0.1/sun.nio.ch.IOUtil.write(IOUtil.java:81)
java.base@20.0.1/sun.nio.ch.IOUtil.write(IOUtil.java:58)
java.base@20.0.1/sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:542)
org.elasticsearch.transport.CopyBytesSocketChannel.writeToSocketChannel(CopyBytesSocketChannel.java:126)
org.elasticsearch.transport.CopyBytesSocketChannel.doWrite(CopyBytesSocketChannel.java:94)
io.netty.channel.AbstractChannel$AbstractUnsafe.flush0(AbstractChannel.java:949)
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.flush0(AbstractNioChannel.java:354)
io.netty.channel.AbstractChannel$AbstractUnsafe.flush(AbstractChannel.java:913)
io.netty.channel.DefaultChannelPipeline$HeadContext.flush(DefaultChannelPipeline.java:1372)
io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:750)
io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:742)
io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:728)
io.netty.handler.ssl.SslHandler.forceFlush(SslHandler.java:2137)
io.netty.handler.ssl.SslHandler.wrapAndFlush(SslHandler.java:804)
io.netty.handler.ssl.SslHandler.flush(SslHandler.java:781)
io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:750)
io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:742)
io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:728)
io.netty.handler.logging.LoggingHandler.flush(LoggingHandler.java:304)
io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:750)
io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:742)
io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:728)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.doFlush(Netty4MessageChannelHandler.java:172)
org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.flush(Netty4MessageChannelHandler.java:110)
io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:750)
io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:765)
io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)
io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)
io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:469)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
77.0% [cpu=76.3%, other=0.7%] (385ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-9][write][T#2]'
2/10 snapshots sharing following 20 elements
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 26 elements
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
6/10 snapshots sharing following 3 elements
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
63.7% [cpu=63.7%, other=0.0%] (318.7ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-9][write][T#5]'
4/10 snapshots sharing following 26 elements
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7350/0x00000008020ea8b8.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 21 elements
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7350/0x00000008020ea8b8.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:88)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7350/0x00000008020ea8b8.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
57.5% [cpu=57.5%, other=0.0%] (287.6ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-9][[alerts-redacted-000134][0]: Lucene Merge Thread #133]'
2/10 snapshots sharing following 15 elements
app//org.apache.lucene.codecs.lucene80.Lucene80NormsProducer$3.longValue(Lucene80NormsProducer.java:349)
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.startDoc(Lucene84PostingsWriter.java:266)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
3/10 snapshots sharing following 13 elements
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:161)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
3/10 snapshots sharing following 17 elements
app//org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:212)
app//org.apache.lucene.codecs.lucene84.ForUtil.encode(ForUtil.java:291)
app//org.apache.lucene.codecs.lucene84.ForDeltaUtil.encodeDeltas(ForDeltaUtil.java:67)
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.startDoc(Lucene84PostingsWriter.java:243)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 11 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
43.8% [cpu=43.1%, other=0.7%] (219ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-9][write][T#3]'
4/10 snapshots sharing following 17 elements
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7350/0x00000008020ea8b8.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 24 elements
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:577)
app//org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:741)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7350/0x00000008020ea8b8.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
22.2% [cpu=22.2%, other=0.0%] (111.2ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-9][write][T#4]'
2/10 snapshots sharing following 23 elements
app//org.elasticsearch.index.mapper.DocumentParser.getLeafMapper(DocumentParser.java:971)
app//org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:739)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
8/10 snapshots sharing following 3 elements
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
22.1% [cpu=22.1%, other=0.0%] (110.6ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-9][[alerts-redacted-000004][0]: Lucene Merge Thread #109]'
7/10 snapshots sharing following 24 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.ByteBuffersDataOutput.copyTo(ByteBuffersDataOutput.java:287)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeBlock(Lucene80DocValuesConsumer.java:367)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValuesMultipleBlocks(Lucene80DocValuesConsumer.java:322)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:284)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedNumericField(Lucene80DocValuesConsumer.java:850)
app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedNumericField(DocValuesConsumer.java:375)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:147)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
3/10 snapshots sharing following 13 elements
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:284)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedNumericField(Lucene80DocValuesConsumer.java:850)
app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedNumericField(DocValuesConsumer.java:375)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:147)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
21.7% [cpu=21.7%, other=0.0%] (108.3ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-9][write][T#8]'
4/10 snapshots sharing following 26 elements
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7350/0x00000008020ea8b8.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 18 elements
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7350/0x00000008020ea8b8.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
4/10 snapshots sharing following 14 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue$Node.block(LinkedTransferQueue.java:470)
java.base@20.0.1/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
java.base@20.0.1/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:669)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:616)
java.base@20.0.1/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1286)
app//org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:152)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3.0% [cpu=2.7%, other=0.3%] (15ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-9][write][T#7]'
6/10 snapshots sharing following 3 elements
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
1.7% [cpu=1.7%, other=0.0%] (8.4ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-9][[alerts-redacted-traffic-000007][0]: Lucene Merge Thread #217]'
10/10 snapshots sharing following 19 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.copyChunks(CompressingStoredFieldsWriter.java:583)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:633)
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:105)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
1.3% [cpu=1.3%, other=0.0%] (6.7ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-9][[alerts-redacted-traffic-000007][0]: Lucene Merge Thread #254]'
10/10 snapshots sharing following 17 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:97)
app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5313)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4821)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
0.0% [cpu=0.0%, other=0.0%] (81.3micros out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-9][refresh][T#1]'
2/10 snapshots sharing following 32 elements
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:120)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:267)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
7/10 snapshots sharing following 3 elements
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (52.2micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'java-sdk-http-connection-reaper'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:188)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 5 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'ml-cpp-log-tail-thread'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.io.FileInputStream.readBytes(Native Method)
java.base@20.0.1/java.io.FileInputStream.read(FileInputStream.java:269)
org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:106)
org.elasticsearch.xpack.ml.process.NativeController.lambda$tailLogsInThread$0(NativeController.java:103)
org.elasticsearch.xpack.ml.process.NativeController$$Lambda$3538/0x0000000801887318.run(Unknown Source)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'process reaper (pid 213)'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.lang.ProcessHandleImpl.waitForProcessExit0(Native Method)
java.base@20.0.1/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:163)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
java.base@20.0.1/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:186)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.17.10]'
10/10 snapshots sharing following 8 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:221)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:754)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1099)
java.base@20.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
::: {alerts-es-hot-8}{72fcD7aiSGm6kR7kReVoYA}{aYytXGIOT3-N7Zhp_cqtDg}{172.16.4.49}{172.16.4.49:9300}{hir}{k8s_node_name=ip-172-16-47-168.us-east-2.compute.internal, xpack.installed=true, data=hot, transform.node=false}
Hot threads at 2023-05-23T17:43:16.191Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
100.0% [cpu=57.4%, other=42.6%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][management][T#5]'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=53.3%, other=46.7%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][write][T#3]'
5/10 snapshots sharing following 33 elements
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:120)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:267)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:380)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:403)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7224/0x00000008020abd30.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 35 elements
app//org.apache.lucene.store.DataInput.readVInt(DataInput.java:129)
app//org.apache.lucene.index.FreqProxFields$FreqProxPostingsEnum.nextPosition(FreqProxFields.java:496)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:150)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:120)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:267)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:380)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:403)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7224/0x00000008020abd30.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 34 elements
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.addPosition(Lucene84PostingsWriter.java:310)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:161)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:120)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:267)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:380)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:403)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7224/0x00000008020abd30.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:957)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:120)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:267)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:380)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:403)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7224/0x00000008020abd30.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=50.6%, other=49.4%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][refresh][T#3]'
10/10 snapshots sharing following 34 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:120)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:267)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=44.1%, other=55.9%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][write][T#5]'
2/10 snapshots sharing following 29 elements
app//org.apache.lucene.analysis.LowerCaseFilter.incrementToken(LowerCaseFilter.java:41)
app//org.apache.lucene.analysis.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:49)
app//org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:924)
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 21 elements
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 22 elements
app//org.elasticsearch.xcontent.json.JsonXContentParser.nextToken(JsonXContentParser.java:73)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:518)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 16 elements
app//org.elasticsearch.action.support.replication.ReplicationOperation$$Lambda$7529/0x00000008020f9220.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:447)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.finishRequest(TransportShardBulkAction.java:233)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:196)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=39.5%, other=60.5%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][write][T#7]'
2/10 snapshots sharing following 28 elements
app//org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:974)
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7224/0x00000008020abd30.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 28 elements
app//org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:924)
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7224/0x00000008020abd30.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
5/10 snapshots sharing following 15 elements
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7224/0x00000008020abd30.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=38.2%, other=61.8%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][write][T#8]'
5/10 snapshots sharing following 27 elements
java.base@20.0.1/sun.nio.ch.UnixFileDispatcherImpl.force0(Native Method)
java.base@20.0.1/sun.nio.ch.UnixFileDispatcherImpl.force(UnixFileDispatcherImpl.java:85)
java.base@20.0.1/sun.nio.ch.FileChannelImpl.force(FileChannelImpl.java:525)
app//org.elasticsearch.index.translog.TranslogWriter.syncUpTo(TranslogWriter.java:481)
app//org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:844)
app//org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:865)
app//org.elasticsearch.index.engine.InternalEngine.ensureTranslogSynced(InternalEngine.java:569)
app//org.elasticsearch.index.shard.IndexShard$7.write(IndexShard.java:3743)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.processList(AsyncIOProcessor.java:97)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.drainAndProcessAndRelease(AsyncIOProcessor.java:85)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.put(AsyncIOProcessor.java:73)
app//org.elasticsearch.index.shard.IndexShard.sync(IndexShard.java:3766)
app//org.elasticsearch.action.support.replication.TransportWriteAction$AsyncAfterWriteAction.run(TransportWriteAction.java:455)
app//org.elasticsearch.action.support.replication.TransportWriteAction$WriteReplicaResult.runPostReplicaActions(TransportWriteAction.java:336)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.lambda$onResponse$2(TransportReplicationAction.java:669)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction$$Lambda$7195/0x00000008020a57d8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:447)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
5/10 snapshots sharing following 6 elements
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=37.2%, other=62.8%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][[alerts-redacted-000059][0]: Lucene Merge Thread #1065]'
2/10 snapshots sharing following 12 elements
app//org.apache.lucene.index.FilterLeafReader$FilterTermsEnum.next(FilterLeafReader.java:194)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:310)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
3/10 snapshots sharing following 15 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.compileIndex(BlockTreeTermsWriter.java:448)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:637)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:947)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
3/10 snapshots sharing following 14 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:637)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:947)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 11 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
100.0% [cpu=35.9%, other=64.1%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][refresh][T#1]'
2/10 snapshots sharing following 34 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:120)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:267)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:350)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:476)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:656)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
8/10 snapshots sharing following 3 elements
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=34.2%, other=65.8%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][write][T#4]'
5/10 snapshots sharing following 11 elements
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 17 elements
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7224/0x00000008020abd30.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 22 elements
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7224/0x00000008020abd30.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=33.3%, other=66.7%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][write][T#6]'
5/10 snapshots sharing following 27 elements
java.base@20.0.1/sun.nio.ch.UnixFileDispatcherImpl.force0(Native Method)
java.base@20.0.1/sun.nio.ch.UnixFileDispatcherImpl.force(UnixFileDispatcherImpl.java:85)
java.base@20.0.1/sun.nio.ch.FileChannelImpl.force(FileChannelImpl.java:525)
app//org.elasticsearch.index.translog.TranslogWriter.syncUpTo(TranslogWriter.java:481)
app//org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:844)
app//org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:865)
app//org.elasticsearch.index.engine.InternalEngine.ensureTranslogSynced(InternalEngine.java:569)
app//org.elasticsearch.index.shard.IndexShard$7.write(IndexShard.java:3743)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.processList(AsyncIOProcessor.java:97)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.drainAndProcessAndRelease(AsyncIOProcessor.java:85)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.put(AsyncIOProcessor.java:73)
app//org.elasticsearch.index.shard.IndexShard.sync(IndexShard.java:3766)
app//org.elasticsearch.action.support.replication.TransportWriteAction$AsyncAfterWriteAction.run(TransportWriteAction.java:455)
app//org.elasticsearch.action.support.replication.TransportWriteAction$WriteReplicaResult.runPostReplicaActions(TransportWriteAction.java:336)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.lambda$onResponse$2(TransportReplicationAction.java:669)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction$$Lambda$7195/0x00000008020a57d8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:447)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 26 elements
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 32 elements
app//org.apache.lucene.util.compress.LZ4.compressWithDictionary(LZ4.java:479)
app//org.apache.lucene.codecs.lucene87.LZ4WithPresetDictCompressionMode$LZ4WithPresetDictCompressor.doCompress(LZ4WithPresetDictCompressionMode.java:162)
app//org.apache.lucene.codecs.lucene87.LZ4WithPresetDictCompressionMode$LZ4WithPresetDictCompressor.compress(LZ4WithPresetDictCompressionMode.java:185)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:260)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:172)
app//org.apache.lucene.index.StoredFieldsConsumer.finishDocument(StoredFieldsConsumer.java:68)
app//org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:463)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:499)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=33.2%, other=66.8%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][write][T#2]'
2/10 snapshots sharing following 30 elements
app//org.apache.lucene.util.BytesRefHash.add(BytesRefHash.java:273)
app//org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:182)
app//org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:974)
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7224/0x00000008020abd30.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 30 elements
app//org.apache.lucene.analysis.LowerCaseFilter.incrementToken(LowerCaseFilter.java:41)
app//org.apache.lucene.analysis.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:49)
app//org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:924)
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7224/0x00000008020abd30.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 21 elements
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7224/0x00000008020abd30.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 27 elements
app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:527)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:491)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$7224/0x00000008020abd30.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
java.base@20.0.1/sun.nio.ch.UnixFileDispatcherImpl.force0(Native Method)
java.base@20.0.1/sun.nio.ch.UnixFileDispatcherImpl.force(UnixFileDispatcherImpl.java:85)
java.base@20.0.1/sun.nio.ch.FileChannelImpl.force(FileChannelImpl.java:525)
app//org.elasticsearch.index.translog.TranslogWriter.syncUpTo(TranslogWriter.java:481)
app//org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:844)
app//org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:865)
app//org.elasticsearch.index.engine.InternalEngine.ensureTranslogSynced(InternalEngine.java:569)
app//org.elasticsearch.index.shard.IndexShard$7.write(IndexShard.java:3743)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.processList(AsyncIOProcessor.java:97)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.drainAndProcessAndRelease(AsyncIOProcessor.java:85)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.put(AsyncIOProcessor.java:73)
app//org.elasticsearch.index.shard.IndexShard.sync(IndexShard.java:3766)
app//org.elasticsearch.action.support.replication.TransportWriteAction$AsyncAfterWriteAction.run(TransportWriteAction.java:455)
app//org.elasticsearch.action.support.replication.TransportWriteAction$WriteReplicaResult.runPostReplicaActions(TransportWriteAction.java:336)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.lambda$onResponse$2(TransportReplicationAction.java:669)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction$$Lambda$7195/0x00000008020a57d8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:447)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=22.9%, other=77.1%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][write][T#1]'
5/10 snapshots sharing following 27 elements
java.base@20.0.1/sun.nio.ch.UnixFileDispatcherImpl.force0(Native Method)
java.base@20.0.1/sun.nio.ch.UnixFileDispatcherImpl.force(UnixFileDispatcherImpl.java:85)
java.base@20.0.1/sun.nio.ch.FileChannelImpl.force(FileChannelImpl.java:525)
app//org.elasticsearch.index.translog.TranslogWriter.syncUpTo(TranslogWriter.java:481)
app//org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:844)
app//org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:865)
app//org.elasticsearch.index.engine.InternalEngine.ensureTranslogSynced(InternalEngine.java:569)
app//org.elasticsearch.index.shard.IndexShard$7.write(IndexShard.java:3743)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.processList(AsyncIOProcessor.java:97)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.drainAndProcessAndRelease(AsyncIOProcessor.java:85)
app//org.elasticsearch.common.util.concurrent.AsyncIOProcessor.put(AsyncIOProcessor.java:73)
app//org.elasticsearch.index.shard.IndexShard.sync(IndexShard.java:3766)
app//org.elasticsearch.action.support.replication.TransportWriteAction$AsyncAfterWriteAction.run(TransportWriteAction.java:455)
app//org.elasticsearch.action.support.replication.TransportWriteAction$WriteReplicaResult.runPostReplicaActions(TransportWriteAction.java:336)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.lambda$onResponse$2(TransportReplicationAction.java:669)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction$$Lambda$7195/0x00000008020a57d8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:447)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 24 elements
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 29 elements
app//org.elasticsearch.xcontent.json.JsonXContentParser.text(JsonXContentParser.java:104)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.textOrNull(AbstractXContentParser.java:241)
app//org.elasticsearch.index.mapper.KeywordFieldMapper.parseCreateField(KeywordFieldMapper.java:557)
app//org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:229)
app//org.elasticsearch.index.mapper.FieldMapper$MultiFields.parse(FieldMapper.java:520)
app//org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:259)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:577)
app//org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:741)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
unique snapshot
app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.getText(UTF8StreamJsonParser.java:334)
app//org.elasticsearch.xcontent.json.JsonXContentParser.text(JsonXContentParser.java:104)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.textOrNull(AbstractXContentParser.java:241)
app//org.elasticsearch.index.mapper.TextFieldMapper.parseCreateField(TextFieldMapper.java:954)
app//org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:229)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:577)
app//org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:741)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=3.3%, other=96.7%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][transport_worker][T#6]'
8/10 snapshots sharing following 4 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=0.5%, other=99.5%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][transport_worker][T#8]'
2/10 snapshots sharing following 8 elements
io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)
io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)
io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:469)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
7/10 snapshots sharing following 10 elements
java.base@20.0.1/sun.nio.ch.EPoll.wait(Native Method)
java.base@20.0.1/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
java.base@20.0.1/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
java.base@20.0.1/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:810)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=0.1%, other=99.9%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][transport_worker][T#3]'
8/10 snapshots sharing following 4 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
97.6% [cpu=34.3%, other=63.3%] (488ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][[alerts-redacted-000145][0]: Lucene Merge Thread #2236]'
3/10 snapshots sharing following 18 elements
app//org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:70)
app//org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:212)
app//org.apache.lucene.codecs.lucene84.ForUtil.encode(ForUtil.java:291)
app//org.apache.lucene.codecs.lucene84.PForUtil.encode(PForUtil.java:93)
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.addPosition(Lucene84PostingsWriter.java:310)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:161)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 16 elements
app//org.apache.lucene.codecs.lucene84.ForUtil.encode(ForUtil.java:291)
app//org.apache.lucene.codecs.lucene84.PForUtil.encode(PForUtil.java:93)
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.addPosition(Lucene84PostingsWriter.java:310)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:161)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
5/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
96.6% [cpu=45.4%, other=51.2%] (483ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][[alerts-redacted-traffic-000052][0]: Lucene Merge Thread #252]'
4/10 snapshots sharing following 24 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.writeBytes(DataOutput.java:52)
app//org.apache.lucene.util.packed.DirectWriter.flush(DirectWriter.java:86)
app//org.apache.lucene.util.packed.DirectWriter.add(DirectWriter.java:78)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.doAddSortedField(Lucene80DocValuesConsumer.java:676)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:891)
app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:804)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:145)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:882)
app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:804)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:145)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
4/10 snapshots sharing following 11 elements
app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:804)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:145)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
94.2% [cpu=47.6%, other=46.6%] (471ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][[alerts-redacted-000074][0]: Lucene Merge Thread #311]'
6/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:891)
app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:804)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:145)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 18 elements
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$21.ordValue(Lucene80DocValuesProducer.java:988)
app//org.apache.lucene.index.SingletonSortedSetDocValues.nextDoc(SingletonSortedSetDocValues.java:66)
app//org.apache.lucene.codecs.DocValuesConsumer$SortedSetDocValuesSub.nextDoc(DocValuesConsumer.java:742)
app//org.apache.lucene.index.DocIDMerger$Sub.nextMappedDoc(DocIDMerger.java:56)
app//org.apache.lucene.index.DocIDMerger$SequentialDocIDMerger.next(DocIDMerger.java:116)
app//org.apache.lucene.codecs.DocValuesConsumer$5$1.nextDoc(DocValuesConsumer.java:848)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:882)
app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:804)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:145)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 9 elements
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
91.4% [cpu=34.2%, other=57.2%] (457ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][[alerts-redacted-traffic-000052][0]: Lucene Merge Thread #251]'
2/10 snapshots sharing following 13 elements
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:146)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
3/10 snapshots sharing following 17 elements
app//org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:213)
app//org.apache.lucene.codecs.lucene84.ForUtil.encode(ForUtil.java:291)
app//org.apache.lucene.codecs.lucene84.PForUtil.encode(PForUtil.java:93)
app//org.apache.lucene.codecs.lucene84.Lucene84PostingsWriter.addPosition(Lucene84PostingsWriter.java:310)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:161)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 13 elements
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:161)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
3/10 snapshots sharing following 17 elements
app//org.apache.lucene.index.MappingMultiPostingsEnum$MappingPostingsSub.nextDoc(MappingMultiPostingsEnum.java:51)
app//org.apache.lucene.index.DocIDMerger$Sub.nextMappedDoc(DocIDMerger.java:56)
app//org.apache.lucene.index.DocIDMerger$SequentialDocIDMerger.next(DocIDMerger.java:116)
app//org.apache.lucene.index.MappingMultiPostingsEnum.nextDoc(MappingMultiPostingsEnum.java:103)
app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:133)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
52.6% [cpu=18.1%, other=34.5%] (263ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][refresh][T#2]'
10/10 snapshots sharing following 31 elements
java.base@20.0.1/java.lang.Object.wait0(Native Method)
java.base@20.0.1/java.lang.Object.wait(Object.java:366)
java.base@20.0.1/java.lang.Object.wait(Object.java:339)
app//org.apache.lucene.index.DocumentsWriterFlushControl.waitForFlush(DocumentsWriterFlushControl.java:283)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:659)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:605)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:173)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:56)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:28)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:370)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:350)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1891)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1870)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3922)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:917)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:102)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1043)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:133)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
16.4% [cpu=1.4%, other=15.0%] (82ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][[alerts-redacted-traffic-000008][0]: Lucene Merge Thread #145]'
10/10 snapshots sharing following 17 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:97)
app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5313)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4821)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
0.0% [cpu=0.0%, other=0.0%] (40.7micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'java-sdk-http-connection-reaper'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:188)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-8][management][T#4]'
unique snapshot
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsReader.ramBytesUsed(BlockTreeTermsReader.java:332)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.ramBytesUsed(PerFieldPostingsFormat.java:358)
app//org.apache.lucene.index.CodecReader.ramBytesUsed(CodecReader.java:214)
app//org.elasticsearch.index.engine.Engine.fillSegmentStats(Engine.java:874)
app//org.elasticsearch.index.engine.Engine.segmentsStats(Engine.java:856)
app//org.elasticsearch.index.shard.IndexShard.segmentStats(IndexShard.java:1387)
app//org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:207)
app//org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:520)
app//org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:474)
app//org.elasticsearch.indices.IndicesService.stats(IndicesService.java:465)
app//org.elasticsearch.node.NodeService.stats(NodeService.java:149)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:87)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:80)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:32)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:333)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:327)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:341)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:404)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:394)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$4(AuthorizationService.java:397)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$6290/0x0000000801f164a8.accept(Unknown Source)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:964)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:928)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$5(AuthorizationService.java:411)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$6292/0x0000000801f168f8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authz.RBACEngine.authorizeClusterAction(RBACEngine.java:178)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:401)
org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:377)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$1(AuthorizationService.java:262)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$6262/0x0000000801f10000.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$1(RBACEngine.java:148)
org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$6264/0x0000000801f10440.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:277)
org.elasticsearch.xpack.security.authz.RBACEngine.getRoles(RBACEngine.java:154)
org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:137)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:264)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:136)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$6159/0x0000000801d9b218.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:102)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:199)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:128)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:415)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:67)
app//org.elasticsearch.transport.InboundHandler$1.doRun(InboundHandler.java:272)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 5 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'ml-cpp-log-tail-thread'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.io.FileInputStream.readBytes(Native Method)
java.base@20.0.1/java.io.FileInputStream.read(FileInputStream.java:269)
org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:106)
org.elasticsearch.xpack.ml.process.NativeController.lambda$tailLogsInThread$0(NativeController.java:103)
org.elasticsearch.xpack.ml.process.NativeController$$Lambda$3540/0x000000080189d1a0.run(Unknown Source)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'process reaper (pid 230)'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.lang.ProcessHandleImpl.waitForProcessExit0(Native Method)
java.base@20.0.1/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:163)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
java.base@20.0.1/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:186)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.17.10]'
10/10 snapshots sharing following 8 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:221)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:754)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1099)
java.base@20.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
::: {alerts-es-hot-2}{mYxNylR7TPuf9rudC-bE5Q}{UVP6jqngSpWtrEwE7Yk-DA}{172.16.21.255}{172.16.21.255:9300}{hir}{k8s_node_name=ip-172-16-26-63.us-east-2.compute.internal, xpack.installed=true, data=hot, transform.node=false}
Hot threads at 2023-05-23T17:43:16.187Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
100.0% [cpu=3.1%, other=96.9%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-2][transport_worker][T#8]'
8/10 snapshots sharing following 4 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=1.9%, other=98.1%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-2][transport_worker][T#4]'
7/10 snapshots sharing following 4 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=1.0%, other=99.0%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-2][transport_worker][T#1]'
unique snapshot
app//com.fasterxml.jackson.core.util.TextBuffer.contentsAsInt(TextBuffer.java:588)
app//com.fasterxml.jackson.core.base.ParserBase._parseNumericValue(ParserBase.java:829)
app//com.fasterxml.jackson.core.base.ParserBase.getLongValue(ParserBase.java:724)
app//org.elasticsearch.xcontent.json.JsonXContentParser.doLongValue(JsonXContentParser.java:187)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.longValue(AbstractXContentParser.java:195)
app//org.elasticsearch.xcontent.support.AbstractXContentParser.longValue(AbstractXContentParser.java:185)
app//org.elasticsearch.action.bulk.BulkRequestParser.parse(BulkRequestParser.java:258)
app//org.elasticsearch.action.bulk.BulkRequest.add(BulkRequest.java:337)
app//org.elasticsearch.rest.action.document.RestBulkAction.prepareRequest(RestBulkAction.java:94)
app//org.elasticsearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:86)
org.elasticsearch.xpack.security.rest.SecurityRestFilter.lambda$handleRequest$0(SecurityRestFilter.java:90)
org.elasticsearch.xpack.security.rest.SecurityRestFilter$$Lambda$6612/0x0000000801fefc90.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authc.support.SecondaryAuthenticator.lambda$authenticateAndAttachToContext$2(SecondaryAuthenticator.java:84)
org.elasticsearch.xpack.security.authc.support.SecondaryAuthenticator$$Lambda$6615/0x0000000801feec50.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authc.support.SecondaryAuthenticator.authenticate(SecondaryAuthenticator.java:94)
org.elasticsearch.xpack.security.authc.support.SecondaryAuthenticator.authenticateAndAttachToContext(SecondaryAuthenticator.java:78)
org.elasticsearch.xpack.security.rest.SecurityRestFilter.lambda$handleRequest$2(SecurityRestFilter.java:85)
org.elasticsearch.xpack.security.rest.SecurityRestFilter$$Lambda$6592/0x0000000801fbdb38.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
app//org.elasticsearch.action.ActionListener$RunBeforeActionListener.onResponse(ActionListener.java:389)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.writeAuthToContext(AuthenticatorChain.java:376)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.finishAuthentication(AuthenticatorChain.java:352)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.maybeLookupRunAsUser(AuthenticatorChain.java:205)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$doAuthenticate$1(AuthenticatorChain.java:128)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6595/0x0000000801fbe3b0.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.core.common.IteratingActionListener.onResponse(IteratingActionListener.java:141)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$getAuthenticatorConsumer$3(AuthenticatorChain.java:175)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6599/0x0000000801fbf078.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator.lambda$consumeToken$3(RealmsAuthenticator.java:219)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator$$Lambda$6602/0x0000000801fbf6e0.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.core.common.IteratingActionListener.onResponse(IteratingActionListener.java:141)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator.lambda$consumeToken$0(RealmsAuthenticator.java:164)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator$$Lambda$6605/0x0000000801fbfb20.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm.lambda$authenticateWithCache$1(CachingUsernamePasswordRealm.java:155)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm$$Lambda$6643/0x0000000801ff0860.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm.handleCachedAuthentication(CachingUsernamePasswordRealm.java:242)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm.lambda$authenticateWithCache$2(CachingUsernamePasswordRealm.java:139)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm$$Lambda$6641/0x0000000801ff0420.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListenerDirectly(ListenableFuture.java:113)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:55)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm.authenticateWithCache(CachingUsernamePasswordRealm.java:134)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm.authenticate(CachingUsernamePasswordRealm.java:105)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator.lambda$consumeToken$2(RealmsAuthenticator.java:148)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator$$Lambda$6601/0x0000000801fbf4b8.accept(Unknown Source)
org.elasticsearch.xpack.core.common.IteratingActionListener.run(IteratingActionListener.java:117)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator.consumeToken(RealmsAuthenticator.java:233)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator.authenticate(RealmsAuthenticator.java:84)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$getAuthenticatorConsumer$5(AuthenticatorChain.java:171)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6597/0x0000000801fbe7f0.accept(Unknown Source)
org.elasticsearch.xpack.core.common.IteratingActionListener.onResponse(IteratingActionListener.java:135)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$getAuthenticatorConsumer$5(AuthenticatorChain.java:165)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6597/0x0000000801fbe7f0.accept(Unknown Source)
org.elasticsearch.xpack.core.common.IteratingActionListener.onResponse(IteratingActionListener.java:135)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$getAuthenticatorConsumer$5(AuthenticatorChain.java:165)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6597/0x0000000801fbe7f0.accept(Unknown Source)
org.elasticsearch.xpack.core.common.IteratingActionListener.onResponse(IteratingActionListener.java:135)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$getAuthenticatorConsumer$5(AuthenticatorChain.java:165)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6597/0x0000000801fbe7f0.accept(Unknown Source)
org.elasticsearch.xpack.core.common.IteratingActionListener.run(IteratingActionListener.java:117)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.doAuthenticate(AuthenticatorChain.java:143)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:104)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:149)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:127)
org.elasticsearch.xpack.security.rest.SecurityRestFilter.handleRequest(SecurityRestFilter.java:79)
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:337)
app//org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:403)
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:255)
app//org.elasticsearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:382)
app//org.elasticsearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:461)
app//org.elasticsearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:357)
org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:35)
org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:19)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.elasticsearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:48)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1374)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1237)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1286)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=0.6%, other=99.4%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-2][transport_worker][T#6]'
unique snapshot
java.base@20.0.1/java.util.ArrayList.add(ArrayList.java:454)
java.base@20.0.1/java.util.ArrayList.add(ArrayList.java:467)
app//org.elasticsearch.action.bulk.BulkRequest.internalAdd(BulkRequest.java:161)
app//org.elasticsearch.action.bulk.BulkRequest$$Lambda$7502/0x0000000801f0b1f8.accept(Unknown Source)
app//org.elasticsearch.action.bulk.BulkRequestParser.parse(BulkRequestParser.java:370)
app//org.elasticsearch.action.bulk.BulkRequest.add(BulkRequest.java:337)
app//org.elasticsearch.rest.action.document.RestBulkAction.prepareRequest(RestBulkAction.java:94)
app//org.elasticsearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:86)
org.elasticsearch.xpack.security.rest.SecurityRestFilter.lambda$handleRequest$0(SecurityRestFilter.java:90)
org.elasticsearch.xpack.security.rest.SecurityRestFilter$$Lambda$6612/0x0000000801fefc90.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authc.support.SecondaryAuthenticator.lambda$authenticateAndAttachToContext$2(SecondaryAuthenticator.java:84)
org.elasticsearch.xpack.security.authc.support.SecondaryAuthenticator$$Lambda$6615/0x0000000801feec50.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authc.support.SecondaryAuthenticator.authenticate(SecondaryAuthenticator.java:94)
org.elasticsearch.xpack.security.authc.support.SecondaryAuthenticator.authenticateAndAttachToContext(SecondaryAuthenticator.java:78)
org.elasticsearch.xpack.security.rest.SecurityRestFilter.lambda$handleRequest$2(SecurityRestFilter.java:85)
org.elasticsearch.xpack.security.rest.SecurityRestFilter$$Lambda$6592/0x0000000801fbdb38.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
app//org.elasticsearch.action.ActionListener$RunBeforeActionListener.onResponse(ActionListener.java:389)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.writeAuthToContext(AuthenticatorChain.java:376)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.finishAuthentication(AuthenticatorChain.java:352)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.maybeLookupRunAsUser(AuthenticatorChain.java:205)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$doAuthenticate$1(AuthenticatorChain.java:128)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6595/0x0000000801fbe3b0.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.core.common.IteratingActionListener.onResponse(IteratingActionListener.java:141)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$getAuthenticatorConsumer$3(AuthenticatorChain.java:175)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6599/0x0000000801fbf078.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator.lambda$consumeToken$3(RealmsAuthenticator.java:219)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator$$Lambda$6602/0x0000000801fbf6e0.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.core.common.IteratingActionListener.onResponse(IteratingActionListener.java:141)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator.lambda$consumeToken$0(RealmsAuthenticator.java:164)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator$$Lambda$6605/0x0000000801fbfb20.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm.lambda$authenticateWithCache$1(CachingUsernamePasswordRealm.java:155)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm$$Lambda$6643/0x0000000801ff0860.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm.handleCachedAuthentication(CachingUsernamePasswordRealm.java:242)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm.lambda$authenticateWithCache$2(CachingUsernamePasswordRealm.java:139)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm$$Lambda$6641/0x0000000801ff0420.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListenerDirectly(ListenableFuture.java:113)
app//org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:55)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm.authenticateWithCache(CachingUsernamePasswordRealm.java:134)
org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm.authenticate(CachingUsernamePasswordRealm.java:105)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator.lambda$consumeToken$2(RealmsAuthenticator.java:148)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator$$Lambda$6601/0x0000000801fbf4b8.accept(Unknown Source)
org.elasticsearch.xpack.core.common.IteratingActionListener.run(IteratingActionListener.java:117)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator.consumeToken(RealmsAuthenticator.java:233)
org.elasticsearch.xpack.security.authc.RealmsAuthenticator.authenticate(RealmsAuthenticator.java:84)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$getAuthenticatorConsumer$5(AuthenticatorChain.java:171)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6597/0x0000000801fbe7f0.accept(Unknown Source)
org.elasticsearch.xpack.core.common.IteratingActionListener.onResponse(IteratingActionListener.java:135)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$getAuthenticatorConsumer$5(AuthenticatorChain.java:165)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6597/0x0000000801fbe7f0.accept(Unknown Source)
org.elasticsearch.xpack.core.common.IteratingActionListener.onResponse(IteratingActionListener.java:135)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$getAuthenticatorConsumer$5(AuthenticatorChain.java:165)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6597/0x0000000801fbe7f0.accept(Unknown Source)
org.elasticsearch.xpack.core.common.IteratingActionListener.onResponse(IteratingActionListener.java:135)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.lambda$getAuthenticatorConsumer$5(AuthenticatorChain.java:165)
org.elasticsearch.xpack.security.authc.AuthenticatorChain$$Lambda$6597/0x0000000801fbe7f0.accept(Unknown Source)
org.elasticsearch.xpack.core.common.IteratingActionListener.run(IteratingActionListener.java:117)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.doAuthenticate(AuthenticatorChain.java:143)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:104)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:149)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:127)
org.elasticsearch.xpack.security.rest.SecurityRestFilter.handleRequest(SecurityRestFilter.java:79)
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:337)
app//org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:403)
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:255)
app//org.elasticsearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:382)
app//org.elasticsearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:461)
app//org.elasticsearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:357)
org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:35)
org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:19)
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
org.elasticsearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:48)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1374)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1237)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1286)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
63.4% [cpu=63.4%, other=0.0%] (316.7ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-2][[alerts-redacted-traffic-000001][0]: Lucene Merge Thread #518]'
2/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 14 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:628)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:947)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
4/10 snapshots sharing following 11 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 12 elements
app//org.apache.lucene.index.FilterLeafReader$FilterTermsEnum.next(FilterLeafReader.java:194)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:310)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
59.3% [cpu=59.3%, other=0.0%] (296.4ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-2][[alerts-redacted-traffic-000007][0]: Lucene Merge Thread #125]'
3/10 snapshots sharing following 17 elements
app//org.apache.lucene.store.BufferedChecksumIndexInput.readBytes(BufferedChecksumIndexInput.java:49)
app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88)
app//org.apache.lucene.store.DataInput.skipBytes(DataInput.java:338)
app//org.apache.lucene.store.ChecksumIndexInput.seek(ChecksumIndexInput.java:52)
app//org.apache.lucene.codecs.CodecUtil.checksumEntireFile(CodecUtil.java:546)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer.checkIntegrity(Lucene80DocValuesProducer.java:1594)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsReader.checkIntegrity(PerFieldDocValuesFormat.java:369)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:131)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 13 elements
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:176)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:126)
app//org.apache.lucene.codecs.DocValuesConsumer.mergeNumericField(DocValuesConsumer.java:179)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:139)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
4/10 snapshots sharing following 13 elements
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:286)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:126)
app//org.apache.lucene.codecs.DocValuesConsumer.mergeNumericField(DocValuesConsumer.java:179)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:139)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
unique snapshot
app//org.apache.lucene.codecs.DocValuesConsumer$1$1.longValue(DocValuesConsumer.java:246)
app//org.apache.lucene.index.SingletonSortedNumericDocValues.nextValue(SingletonSortedNumericDocValues.java:73)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:178)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:126)
app//org.apache.lucene.codecs.DocValuesConsumer.mergeNumericField(DocValuesConsumer.java:179)
app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:139)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:155)
app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:195)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:150)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
32.6% [cpu=31.1%, other=1.5%] (163ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-2][write][T#4]'
2/10 snapshots sharing following 3 elements
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
31.2% [cpu=31.2%, other=0.0%] (156.1ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-2][[alerts-redacted-000129][0]: Lucene Merge Thread #880]'
5/10 snapshots sharing following 22 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.ByteBuffersDataOutput.copyTo(ByteBuffersDataOutput.java:287)
app//org.apache.lucene.codecs.lucene87.LZ4WithPresetDictCompressionMode$LZ4WithPresetDictCompressor.compress(LZ4WithPresetDictCompressionMode.java:189)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:260)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:172)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.copyOneDoc(CompressingStoredFieldsWriter.java:518)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:636)
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:105)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 14 elements
app//org.apache.lucene.codecs.lucene87.LZ4WithPresetDictCompressionMode$LZ4WithPresetDictDecompressor.decompress(LZ4WithPresetDictCompressionMode.java:129)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader$BlockState.doReset(CompressingStoredFieldsReader.java:564)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader$BlockState.reset(CompressingStoredFieldsReader.java:466)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.document(CompressingStoredFieldsReader.java:656)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.copyOneDoc(CompressingStoredFieldsWriter.java:514)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:636)
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:105)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
3/10 snapshots sharing following 15 elements
app//org.apache.lucene.util.compress.LZ4.compressWithDictionary(LZ4.java:479)
app//org.apache.lucene.codecs.lucene87.LZ4WithPresetDictCompressionMode$LZ4WithPresetDictCompressor.doCompress(LZ4WithPresetDictCompressionMode.java:162)
app//org.apache.lucene.codecs.lucene87.LZ4WithPresetDictCompressionMode$LZ4WithPresetDictCompressor.compress(LZ4WithPresetDictCompressionMode.java:185)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:260)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:172)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.copyOneDoc(CompressingStoredFieldsWriter.java:518)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:636)
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:105)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
14.2% [cpu=14.0%, other=0.2%] (71ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-2][write][T#6]'
8/10 snapshots sharing following 3 elements
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
7.0% [cpu=7.0%, other=0.0%] (35.1ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-2][write][T#5]'
2/10 snapshots sharing following 24 elements
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
6/10 snapshots sharing following 20 elements
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2/10 snapshots sharing following 14 elements
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:900)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:320)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:181)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:245)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:134)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:196)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
1.9% [cpu=1.9%, other=0.0%] (9.4ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-2][[associations-000905][0]: Lucene Merge Thread #53203]'
10/10 snapshots sharing following 17 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:97)
app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5313)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4821)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
0.0% [cpu=0.0%, other=0.0%] (24.1micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'java-sdk-http-connection-reaper'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:188)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 5 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'ml-cpp-log-tail-thread'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.io.FileInputStream.readBytes(Native Method)
java.base@20.0.1/java.io.FileInputStream.read(FileInputStream.java:269)
org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:106)
org.elasticsearch.xpack.ml.process.NativeController.lambda$tailLogsInThread$0(NativeController.java:103)
org.elasticsearch.xpack.ml.process.NativeController$$Lambda$3538/0x000000080183bcd0.run(Unknown Source)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'process reaper (pid 214)'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.lang.ProcessHandleImpl.waitForProcessExit0(Native Method)
java.base@20.0.1/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:163)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
java.base@20.0.1/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:186)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.17.10]'
10/10 snapshots sharing following 8 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:221)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:754)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1099)
java.base@20.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
::: {alerts-es-hot-11}{WrmBxHXOS3Cqqg59fm3snw}{kJk4btfZSC-GpwXfgNVjjw}{172.16.45.112}{172.16.45.112:9300}{hir}{k8s_node_name=ip-172-16-63-47.us-east-2.compute.internal, xpack.installed=true, data=hot, transform.node=false}
Hot threads at 2023-05-23T17:43:16.187Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
100.0% [cpu=5.5%, other=94.5%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-11][transport_worker][T#6]'
10/10 snapshots sharing following 4 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=2.7%, other=97.3%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-11][transport_worker][T#7]'
unique snapshot
java.base@20.0.1/sun.nio.ch.SocketDispatcher.read0(Native Method)
java.base@20.0.1/sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:47)
java.base@20.0.1/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:340)
java.base@20.0.1/sun.nio.ch.IOUtil.read(IOUtil.java:294)
java.base@20.0.1/sun.nio.ch.IOUtil.read(IOUtil.java:269)
java.base@20.0.1/sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:425)
org.elasticsearch.transport.CopyBytesSocketChannel.readFromSocketChannel(CopyBytesSocketChannel.java:131)
org.elasticsearch.transport.CopyBytesSocketChannel.doReadBytes(CopyBytesSocketChannel.java:116)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719)
io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
100.0% [cpu=2.1%, other=97.9%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-11][transport_worker][T#3]'
6/10 snapshots sharing following 4 elements
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
50.1% [cpu=50.1%, other=0.0%] (250.3ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-11][[alerts-redacted-traffic-000008][0]: Lucene Merge Thread #74]'
5/10 snapshots sharing following 12 elements
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:907)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
2/10 snapshots sharing following 20 elements
app//org.apache.lucene.store.RAMOutputStream.writeByte(RAMOutputStream.java:137)
app//org.apache.lucene.util.compress.LZ4.encodeLiterals(LZ4.java:151)
app//org.apache.lucene.util.compress.LZ4.encodeLastLiterals(LZ4.java:164)
app//org.apache.lucene.util.compress.LZ4.compressWithDictionary(LZ4.java:488)
app//org.apache.lucene.util.compress.LZ4.compress(LZ4.java:419)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlock(BlockTreeTermsWriter.java:823)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:606)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:947)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:912)
app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:318)
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
3/10 snapshots sharing following 10 elements
app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
17.7% [cpu=17.7%, other=0.0%] (88.5ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-11][write][T#6]'
2/10 snapshots sharing following 28 elements
app//org.elasticsearch.index.mapper.DocumentParser.getLeafMapper(DocumentParser.java:971)
app//org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:739)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:574)
app//org.elasticsearch.index.mapper.DocumentParser.parseObject(DocumentParser.java:608)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:502)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$6931/0x000000080207cd40.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
8/10 snapshots sharing following 3 elements
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
2.8% [cpu=2.7%, other=0.1%] (14ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-11][write][T#7]'
9/10 snapshots sharing following 15 elements
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$6931/0x000000080207cd40.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
1.8% [cpu=1.8%, other=0.0%] (8.9ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-11][write][T#2]'
2/10 snapshots sharing following 25 elements
app//org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:229)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:577)
app//org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:741)
app//org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:516)
app//org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:477)
app//org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:145)
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:91)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:83)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:1025)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:971)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$6931/0x000000080207cd40.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
3/10 snapshots sharing following 25 elements
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
app//org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
app//org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1400)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1312)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1248)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:1051)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:1066)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:998)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:923)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:592)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:564)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.lambda$dispatchedShardOperationOnReplica$4(TransportShardBulkAction.java:526)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$$Lambda$6931/0x000000080207cd40.get(Unknown Source)
app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:436)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:525)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnReplica(TransportShardBulkAction.java:74)
app//org.elasticsearch.action.support.replication.TransportWriteAction$2.doRun(TransportWriteAction.java:224)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
1.3% [cpu=1.3%, other=0.0%] (6.7ms out of 500ms) cpu usage by thread 'elasticsearch[alerts-es-hot-11][[alerts-redacted-000056][0]: Lucene Merge Thread #1117]'
10/10 snapshots sharing following 19 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:164)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.copyChunks(CompressingStoredFieldsWriter.java:583)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:633)
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:105)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4757)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4361)
app//org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5920)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:626)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:684)
0.0% [cpu=0.0%, other=0.0%] (31micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'java-sdk-http-connection-reaper'
10/10 snapshots sharing following 3 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:188)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 5 elements
java.base@20.0.1/java.lang.Thread.sleep0(Native Method)
java.base@20.0.1/java.lang.Thread.sleep(Thread.java:484)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'ml-cpp-log-tail-thread'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.io.FileInputStream.readBytes(Native Method)
java.base@20.0.1/java.io.FileInputStream.read(FileInputStream.java:269)
org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:106)
org.elasticsearch.xpack.ml.process.NativeController.lambda$tailLogsInThread$0(NativeController.java:103)
org.elasticsearch.xpack.ml.process.NativeController$$Lambda$3524/0x00000008018991a0.run(Unknown Source)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'process reaper (pid 214)'
10/10 snapshots sharing following 7 elements
java.base@20.0.1/java.lang.ProcessHandleImpl.waitForProcessExit0(Native Method)
java.base@20.0.1/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:163)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
java.base@20.0.1/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:186)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.17.10]'
10/10 snapshots sharing following 8 elements
java.base@20.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@20.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:221)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:754)
java.base@20.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1099)
java.base@20.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment