Skip to content

Instantly share code, notes, and snippets.

@rex-remind101
Created January 15, 2021 01:08
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save rex-remind101/b6b1a8d1b82d601d343203b0ea44add2 to your computer and use it in GitHub Desktop.
Save rex-remind101/b6b1a8d1b82d601d343203b0ea44add2 to your computer and use it in GitHub Desktop.
hot_threads ES
# Attempt 1
[hadoop@ip-10-128-4-195 ~]$ curl <host>/_nodes/hot_threads
::: {741248536f8a624a46358affbf3940d9}{lunuJX_4QM-QSGxUuAK6aQ}{K6N5Dqy9TUOjP5v5nWfonw}{x.x.x.x}{x.x.x.x:9300}{dir}{distributed_snapshot_deletion_enabled=true, zone=us-east-1d, cross_cluster_transport_address=x.x.x.x, }
Hot threads at 2021-01-15T00:58:24.674Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
76.1% (380.4ms out of 500ms) cpu usage by thread 'elasticsearch[741248536f8a624a46358affbf3940d9][refresh][T#2]'
5/10 snapshots sharing following 36 elements
app//org.apache.lucene.index.PointValuesWriter$1.intersect(PointValuesWriter.java:89)
app//org.apache.lucene.index.PointValuesWriter$MutableSortingPointValues.intersect(PointValuesWriter.java:203)
app//org.apache.lucene.util.bkd.BKDWriter.writeField1Dim(BKDWriter.java:466)
app//org.apache.lucene.util.bkd.BKDWriter.writeField(BKDWriter.java:384)
app//org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.writeField(Lucene60PointsWriter.java:104)
app//org.apache.lucene.index.PointValuesWriter.flush(PointValuesWriter.java:188)
app//org.apache.lucene.index.DefaultIndexingChain.writePoints(DefaultIndexingChain.java:218)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:144)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
5/10 snapshots sharing following 37 elements
app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:128)
app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:121)
app//org.apache.lucene.util.bkd.MutablePointsReaderUtils.sort(MutablePointsReaderUtils.java:91)
app//org.apache.lucene.util.bkd.BKDWriter.writeField1Dim(BKDWriter.java:462)
app//org.apache.lucene.util.bkd.BKDWriter.writeField(BKDWriter.java:384)
app//org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.writeField(Lucene60PointsWriter.java:104)
app//org.apache.lucene.index.PointValuesWriter.flush(PointValuesWriter.java:188)
app//org.apache.lucene.index.DefaultIndexingChain.writePoints(DefaultIndexingChain.java:218)
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:144)
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
0.0% (103.7micros out of 500ms) cpu usage by thread 'MetricAggregationClientPublisher-1'
10/10 snapshots sharing following 13 elements
java.base@11.0.6/jdk.internal.misc.Unsafe.park(Native Method)
java.base@11.0.6/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
java.base@11.0.6/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:513)
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:675)
[AMAZON INTERNAL]
[AMAZON INTERNAL]
java.base@11.0.6/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
java.base@11.0.6/java.util.concurrent.FutureTask.run(FutureTask.java:264)
java.base@11.0.6/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
::: {428abb0d265d296a6b15e382621d60fd}{ToM_ylKTSVejdIAfH_O5DA}{e_ITHIwCTCGaBJVmTmn-YQ}{x.x.x.x}{x.x.x.x:9300}{dir}{distributed_snapshot_deletion_enabled=true, zone=us-east-1b, cross_cluster_transport_address=x.x.x.x}
Hot threads at 2021-01-15T00:58:24.675Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
27.3% (136.6ms out of 500ms) cpu usage by thread 'qtp1356445494-244187'
4/10 snapshots sharing following 47 elements
app//com.fasterxml.jackson.core.json.UTF8JsonGenerator.writeNumber(UTF8JsonGenerator.java:958)
app//com.fasterxml.jackson.core.JsonGenerator._copyCurrentContents(JsonGenerator.java:1975)
app//com.fasterxml.jackson.core.JsonGenerator.copyCurrentStructure(JsonGenerator.java:1914)
app//org.elasticsearch.common.xcontent.json.JsonXContentGenerator.copyCurrentStructure(JsonXContentGenerator.java:418)
app//org.elasticsearch.common.xcontent.XContentBuilder.copyCurrentStructure(XContentBuilder.java:997)
app//org.elasticsearch.action.update.UpdateRequest.lambda$static$3(UpdateRequest.java:82)
app//org.elasticsearch.action.update.UpdateRequest$$Lambda$3431/0x000000080177a040.parse(Unknown Source)
app//org.elasticsearch.common.xcontent.AbstractObjectParser.lambda$declareObject$1(AbstractObjectParser.java:169)
app//org.elasticsearch.common.xcontent.AbstractObjectParser$$Lambda$298/0x00000008002c4040.parse(Unknown Source)
app//org.elasticsearch.common.xcontent.ObjectParser.lambda$declareField$9(ObjectParser.java:386)
app//org.elasticsearch.common.xcontent.ObjectParser$$Lambda$286/0x00000008002c1040.parse(Unknown Source)
app//org.elasticsearch.common.xcontent.ObjectParser.parseValue(ObjectParser.java:529)
app//org.elasticsearch.common.xcontent.ObjectParser.parseSub(ObjectParser.java:541)
app//org.elasticsearch.common.xcontent.ObjectParser.parse(ObjectParser.java:324)
app//org.elasticsearch.action.update.UpdateRequest.fromXContent(UpdateRequest.java:851)
app//org.elasticsearch.action.bulk.BulkRequestParser.parse(BulkRequestParser.java:290)
app//org.elasticsearch.action.bulk.BulkRequest.add(BulkRequest.java:304)
app//org.elasticsearch.rest.action.document.RestBulkAction.prepareRequest(RestBulkAction.java:101)
app//org.elasticsearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:94)
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:236)
app//org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:318)
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:176)
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
app//[AMAZON INTERNAL]
app//org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52)
app//[AMAZON INTERNAL]
app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
app//org.eclipse.jetty.server.Server.handle(Server.java:370)
app//org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
app//org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:960)
app//org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1021)
app//org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865)
app//org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
app//org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
app//org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668)
app//org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
app//org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
2/10 snapshots sharing following 48 elements
app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser._nextTokenNotInObject(UTF8StreamJsonParser.java:855)
app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextToken(UTF8StreamJsonParser.java:754)
app//com.fasterxml.jackson.core.JsonGenerator._copyCurrentContents(JsonGenerator.java:1935)
app//com.fasterxml.jackson.core.JsonGenerator.copyCurrentStructure(JsonGenerator.java:1914)
app//org.elasticsearch.common.xcontent.json.JsonXContentGenerator.copyCurrentStructure(JsonXContentGenerator.java:418)
app//org.elasticsearch.common.xcontent.XContentBuilder.copyCurrentStructure(XContentBuilder.java:997)
app//org.elasticsearch.action.update.UpdateRequest.lambda$static$5(UpdateRequest.java:88)
app//org.elasticsearch.action.update.UpdateRequest$$Lambda$3433/0x000000080177a840.parse(Unknown Source)
app//org.elasticsearch.common.xcontent.AbstractObjectParser.lambda$declareObject$1(AbstractObjectParser.java:169)
app//org.elasticsearch.common.xcontent.AbstractObjectParser$$Lambda$298/0x00000008002c4040.parse(Unknown Source)
app//org.elasticsearch.common.xcontent.ObjectParser.lambda$declareField$9(ObjectParser.java:386)
app//org.elasticsearch.common.xcontent.ObjectParser$$Lambda$286/0x00000008002c1040.parse(Unknown Source)
app//org.elasticsearch.common.xcontent.ObjectParser.parseValue(ObjectParser.java:529)
app//org.elasticsearch.common.xcontent.ObjectParser.parseSub(ObjectParser.java:541)
app//org.elasticsearch.common.xcontent.ObjectParser.parse(ObjectParser.java:324)
app//org.elasticsearch.action.update.UpdateRequest.fromXContent(UpdateRequest.java:851)
app//org.elasticsearch.action.bulk.BulkRequestParser.parse(BulkRequestParser.java:290)
app//org.elasticsearch.action.bulk.BulkRequest.add(BulkRequest.java:304)
app//org.elasticsearch.rest.action.document.RestBulkAction.prepareRequest(RestBulkAction.java:101)
app//org.elasticsearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:94)
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:236)
app//org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:318)
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:176)
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
app//[AMAZON INTERNAL]
app//org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52)
app//[AMAZON INTERNAL]
app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
app//org.eclipse.jetty.server.Server.handle(Server.java:370)
app//org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
app//org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:960)
app//org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1021)
app//org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865)
app//org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
app//org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
app//org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668)
app//org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
app//org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
2/10 snapshots sharing following 45 elements
app//com.fasterxml.jackson.core.JsonGenerator.copyCurrentStructure(JsonGenerator.java:1914)
app//org.elasticsearch.common.xcontent.json.JsonXContentGenerator.copyCurrentStructure(JsonXContentGenerator.java:418)
app//org.elasticsearch.common.xcontent.XContentBuilder.copyCurrentStructure(XContentBuilder.java:997)
app//org.elasticsearch.action.update.UpdateRequest.lambda$static$3(UpdateRequest.java:82)
app//org.elasticsearch.action.update.UpdateRequest$$Lambda$3431/0x000000080177a040.parse(Unknown Source)
app//org.elasticsearch.common.xcontent.AbstractObjectParser.lambda$declareObject$1(AbstractObjectParser.java:169)
app//org.elasticsearch.common.xcontent.AbstractObjectParser$$Lambda$298/0x00000008002c4040.parse(Unknown Source)
app//org.elasticsearch.common.xcontent.ObjectParser.lambda$declareField$9(ObjectParser.java:386)
app//org.elasticsearch.common.xcontent.ObjectParser$$Lambda$286/0x00000008002c1040.parse(Unknown Source)
app//org.elasticsearch.common.xcontent.ObjectParser.parseValue(ObjectParser.java:529)
app//org.elasticsearch.common.xcontent.ObjectParser.parseSub(ObjectParser.java:541)
app//org.elasticsearch.common.xcontent.ObjectParser.parse(ObjectParser.java:324)
app//org.elasticsearch.action.update.UpdateRequest.fromXContent(UpdateRequest.java:851)
app//org.elasticsearch.action.bulk.BulkRequestParser.parse(BulkRequestParser.java:290)
app//org.elasticsearch.action.bulk.BulkRequest.add(BulkRequest.java:304)
app//org.elasticsearch.rest.action.document.RestBulkAction.prepareRequest(RestBulkAction.java:101)
app//org.elasticsearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:94)
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:236)
app//org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:318)
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:176)
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//[AMAZON INTERNAL]
app//org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
app//[AMAZON INTERNAL]
app//org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52)
app//[AMAZON INTERNAL]
app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
app//org.eclipse.jetty.server.Server.handle(Server.java:370)
app//org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
app//org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:960)
app//org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1021)
app//org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865)
app//org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
app//org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
app//org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668)
app//org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
app//org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
2/10 snapshots sharing following 1 elements
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
::: {c9866516a52bdf0eb71c2ed62b04fa13}{X5AJ7ng4SZq9ZwCHRm4TyA}{8eWWFnuPQ3yIaCvSvBW7LA}{x.x.x.x}{x.x.x.x:9300}{mr}{distributed_snapshot_deletion_enabled=true, zone=us-east-1d, cross_cluster_transport_address=x.x.x.x}
Hot threads at 2021-01-15T00:58:24.675Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
0.0% (92.5micros out of 500ms) cpu usage by thread 'MetricAggregationClientPublisher-1'
10/10 snapshots sharing following 13 elements
java.base@11.0.6/jdk.internal.misc.Unsafe.park(Native Method)
java.base@11.0.6/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
java.base@11.0.6/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:513)
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:675)
[AMAZON INTERNAL]
[AMAZON INTERNAL]
java.base@11.0.6/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
java.base@11.0.6/java.util.concurrent.FutureTask.run(FutureTask.java:264)
java.base@11.0.6/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
::: {a253de2f08f5d6b1348f293c60376d38}{yJuxMUq4QN-es3z064UIEw}{hmZe3HskR5GaZ8eoPGMS4w}{x.x.x.x}{x.x.x.x:9300}{dir}{distributed_snapshot_deletion_enabled=true, zone=us-east-1a, cross_cluster_transport_address=x.x.x.x}
Hot threads at 2021-01-15T00:58:24.675Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
0.1% (251.4micros out of 500ms) cpu usage by thread 'MetricAggregationClientPublisher-1'
10/10 snapshots sharing following 13 elements
java.base@11.0.6/jdk.internal.misc.Unsafe.park(Native Method)
java.base@11.0.6/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
java.base@11.0.6/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:513)
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:675)
[AMAZON INTERNAL]
[AMAZON INTERNAL]
java.base@11.0.6/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
java.base@11.0.6/java.util.concurrent.FutureTask.run(FutureTask.java:264)
java.base@11.0.6/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
::: {c1715d6e1bcc3d0e71d0c8a5bc0082ef}{BuCONqzKT8yNJfaebdT_CA}{hatq8-wvSe2dOG8Ynttx8A}{x.x.x.x}{x.x.x.x:9300}{mr}{distributed_snapshot_deletion_enabled=true, zone=us-east-1a, cross_cluster_transport_address=x.x.x.x}
Hot threads at 2021-01-15T00:58:24.690Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
0.0% (90.4micros out of 500ms) cpu usage by thread 'MetricAggregationClientPublisher-1'
10/10 snapshots sharing following 13 elements
java.base@11.0.6/jdk.internal.misc.Unsafe.park(Native Method)
java.base@11.0.6/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
java.base@11.0.6/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:513)
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:675)
[AMAZON INTERNAL]
[AMAZON INTERNAL]
java.base@11.0.6/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
java.base@11.0.6/java.util.concurrent.FutureTask.run(FutureTask.java:264)
java.base@11.0.6/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
::: {3b82871f297f3450044d713d34411d6e}{ftE1pCbqR96KoPMpx2Tl-A}{InjOcO1UTaCVsJZFpLdaow}{x.x.x.x}{x.x.x.x:9300}{mr}{distributed_snapshot_deletion_enabled=true, zone=us-east-1b, cross_cluster_transport_address=x.x.x.x}
Hot threads at 2021-01-15T00:58:24.675Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
1.1% (5.5ms out of 500ms) cpu usage by thread 'elasticsearch[3b82871f297f3450044d713d34411d6e][generic][T#192]'
10/10 snapshots sharing following 55 elements
app//org.conscrypt.NativeCrypto.SSL_read(Native Method)
app//org.conscrypt.NativeSsl.read(NativeSsl.java:409)
app//org.conscrypt.ConscryptFileDescriptorSocket$SSLInputStream.read(ConscryptFileDescriptorSocket.java:548)
app//org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
app//org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
app//org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
app//org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
app//org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
app//org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
[AMAZON INTERNAL]
app//org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
[AMAZON INTERNAL]
[AMAZON INTERNAL]
[AMAZON INTERNAL]
[AMAZON INTERNAL]
[AMAZON INTERNAL]
[AMAZON INTERNAL]
[AMAZON INTERNAL]
[AMAZON INTERNAL]
[AMAZON INTERNAL]
[AMAZON INTERNAL]
[AMAZON INTERNAL]
[AMAZON INTERNAL]
[AMAZON INTERNAL]
org.elasticsearch.repositories.s3.S3RetryingInputStream.lambda$openStream$0(S3RetryingInputStream.java:97)
org.elasticsearch.repositories.s3.S3RetryingInputStream$$Lambda$3555/0x000000010082f440.run(Unknown Source)
java.base@11.0.6/java.security.AccessController.doPrivileged(Native Method)
org.elasticsearch.repositories.s3.SocketAccess.doPrivileged(SocketAccess.java:42)
org.elasticsearch.repositories.s3.S3RetryingInputStream.openStream(S3RetryingInputStream.java:97)
org.elasticsearch.repositories.s3.S3RetryingInputStream.<init>(S3RetryingInputStream.java:85)
org.elasticsearch.repositories.s3.S3RetryingInputStream.<init>(S3RetryingInputStream.java:68)
org.elasticsearch.repositories.s3.S3BlobContainer.readBlob(S3BlobContainer.java:99)
app//org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat.readBlob(ChecksumBlobStoreFormat.java:150)
app//org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat.read(ChecksumBlobStoreFormat.java:136)
app//org.elasticsearch.repositories.blobstore.BlobStoreRepository.getSnapshotInfo(BlobStoreRepository.java:1308)
app//org.elasticsearch.action.admin.cluster.snapshots.get.TransportGetSnapshotsAction.snapshots(TransportGetSnapshotsAction.java:212)
app//org.elasticsearch.action.admin.cluster.snapshots.get.TransportGetSnapshotsAction.masterOperation(TransportGetSnapshotsAction.java:151)
app//org.elasticsearch.action.admin.cluster.snapshots.get.TransportGetSnapshotsAction.masterOperation(TransportGetSnapshotsAction.java:67)
app//org.elasticsearch.action.support.master.TransportMasterNodeAction.masterOperation(TransportMasterNodeAction.java:100)
app//org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.lambda$doStart$3(TransportMasterNodeAction.java:171)
app//org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$$Lambda$3216/0x0000000101704440.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:704)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
# Attempt 2
curl <host>/_nodes/hot_threads
::: {c9866516a52bdf0eb71c2ed62b04fa13}{X5AJ7ng4SZq9ZwCHRm4TyA}{8eWWFnuPQ3yIaCvSvBW7LA}{x.x.x.x}{x.x.x.x:9300}{mr}{distributed_snapshot_deletion_enabled=true, zone=us-east-1d, cross_cluster_transport_address=x.x.x.x}
Hot threads at 2021-01-15T01:04:30.188Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
0.0% (98.6micros out of 500ms) cpu usage by thread 'MetricAggregationClientPublisher-1'
10/10 snapshots sharing following 13 elements
java.base@11.0.6/jdk.internal.misc.Unsafe.park(Native Method)
java.base@11.0.6/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
java.base@11.0.6/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:513)
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:675)
[AMAZON INTERNAL]
[AMAZON INTERNAL]
java.base@11.0.6/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
java.base@11.0.6/java.util.concurrent.FutureTask.run(FutureTask.java:264)
java.base@11.0.6/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
::: {428abb0d265d296a6b15e382621d60fd}{ToM_ylKTSVejdIAfH_O5DA}{e_ITHIwCTCGaBJVmTmn-YQ}{x.x.x.x}{x.x.x.x:9300}{dir}{distributed_snapshot_deletion_enabled=true, zone=us-east-1b, cross_cluster_transport_address=x.x.x.x, }
Hot threads at 2021-01-15T01:04:30.187Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
::: {3b82871f297f3450044d713d34411d6e}{ftE1pCbqR96KoPMpx2Tl-A}{InjOcO1UTaCVsJZFpLdaow}{x.x.x.x}{x.x.x.x:9300}{mr}{distributed_snapshot_deletion_enabled=true, zone=us-east-1b, cross_cluster_transport_address=x.x.x.x}
Hot threads at 2021-01-15T01:04:30.187Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
::: {c1715d6e1bcc3d0e71d0c8a5bc0082ef}{BuCONqzKT8yNJfaebdT_CA}{hatq8-wvSe2dOG8Ynttx8A}{x.x.x.x}{x.x.x.x:9300}{mr}{distributed_snapshot_deletion_enabled=true, zone=us-east-1a, cross_cluster_transport_address=x.x.x.x}
Hot threads at 2021-01-15T01:04:30.187Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
::: {a253de2f08f5d6b1348f293c60376d38}{yJuxMUq4QN-es3z064UIEw}{hmZe3HskR5GaZ8eoPGMS4w}{x.x.x.x}{x.x.x.x:9300}{dir}{distributed_snapshot_deletion_enabled=true, zone=us-east-1a, cross_cluster_transport_address=x.x.x.x}
Hot threads at 2021-01-15T01:04:30.188Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
89.9% (449.3ms out of 500ms) cpu usage by thread 'elasticsearch[a253de2f08f5d6b1348f293c60376d38][write][T#1]'
2/10 snapshots sharing following 40 elements
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:71)
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:267)
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:841)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:818)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:790)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:267)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:157)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:202)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:114)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:81)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:932)
app//org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:109)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.runWithPrimaryShardReference(TransportReplicationAction.java:397)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.lambda$doRun$0(TransportReplicationAction.java:320)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction$$Lambda$3945/0x0000000801a3ac40.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
app//org.elasticsearch.index.shard.IndexShard.lambda$wrapPrimaryOperationPermitListener$24(IndexShard.java:2860)
app//org.elasticsearch.index.shard.IndexShard$$Lambda$3947/0x0000000801a3b440.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:113)
app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:285)
app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:237)
app//org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationPermit(IndexShard.java:2834)
app//org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryOperationPermit(TransportReplicationAction.java:873)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:316)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.action.support.replication.TransportReplicationAction.handlePrimaryRequest(TransportReplicationAction.java:279)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$$Lambda$2445/0x0000000800d12c40.messageReceived(Unknown Source)
[AMAZON INTERNAL]
[AMAZON INTERNAL]
[AMAZON INTERNAL]
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
app//org.elasticsearch.transport.TransportService$8.doRun(TransportService.java:801)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:704)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
4/10 snapshots sharing following 46 elements
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:406)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:250)
app//org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:495)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1594)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1171)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1108)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:948)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:861)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:833)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:790)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:267)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:157)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:202)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:114)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:81)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:932)
app//org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:109)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.runWithPrimaryShardReference(TransportReplicationAction.java:397)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.lambda$doRun$0(TransportReplicationAction.java:320)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction$$Lambda$3945/0x0000000801a3ac40.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
app//org.elasticsearch.index.shard.IndexShard.lambda$wrapPrimaryOperationPermitListener$24(IndexShard.java:2860)
app//org.elasticsearch.index.shard.IndexShard$$Lambda$3947/0x0000000801a3b440.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:113)
app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:285)
app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:237)
app//org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationPermit(IndexShard.java:2834)
app//org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryOperationPermit(TransportReplicationAction.java:873)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:316)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.action.support.replication.TransportReplicationAction.handlePrimaryRequest(TransportReplicationAction.java:279)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$$Lambda$2445/0x0000000800d12c40.messageReceived(Unknown Source)
[AMAZON INTERNAL]
[AMAZON INTERNAL]
[AMAZON INTERNAL]
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
app//org.elasticsearch.transport.TransportService$8.doRun(TransportService.java:801)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:704)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
4/10 snapshots sharing following 52 elements
app//org.apache.lucene.util.compress.LZ4.compress(LZ4.java:441)
app//org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:165)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:229)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:159)
app//org.apache.lucene.index.StoredFieldsConsumer.finishDocument(StoredFieldsConsumer.java:59)
app//org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:378)
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:414)
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:250)
app//org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:495)
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1594)
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1171)
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1108)
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:948)
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:861)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:833)
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:790)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:267)
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:157)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:202)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:114)
app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:81)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:932)
app//org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:109)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.runWithPrimaryShardReference(TransportReplicationAction.java:397)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.lambda$doRun$0(TransportReplicationAction.java:320)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction$$Lambda$3945/0x0000000801a3ac40.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
app//org.elasticsearch.index.shard.IndexShard.lambda$wrapPrimaryOperationPermitListener$24(IndexShard.java:2860)
app//org.elasticsearch.index.shard.IndexShard$$Lambda$3947/0x0000000801a3b440.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:113)
app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:285)
app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:237)
app//org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationPermit(IndexShard.java:2834)
app//org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryOperationPermit(TransportReplicationAction.java:873)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:316)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.action.support.replication.TransportReplicationAction.handlePrimaryRequest(TransportReplicationAction.java:279)
app//org.elasticsearch.action.support.replication.TransportReplicationAction$$Lambda$2445/0x0000000800d12c40.messageReceived(Unknown Source)
[AMAZON INTERNAL]
[AMAZON INTERNAL]
[AMAZON INTERNAL]
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
app//org.elasticsearch.transport.TransportService$8.doRun(TransportService.java:801)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:704)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
82.9% (414.3ms out of 500ms) cpu usage by thread 'elasticsearch[a253de2f08f5d6b1348f293c60376d38][[people][3]: Lucene Merge Thread #17542]'
2/10 snapshots sharing following 11 elements
app//org.apache.lucene.util.PriorityQueue.downHeap(PriorityQueue.java:267)
app//org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:202)
app//org.apache.lucene.util.bkd.BKDWriter.merge(BKDWriter.java:518)
app//org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:213)
app//org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:202)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:162)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
2/10 snapshots sharing following 22 elements
java.base@11.0.6/sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:280)
java.base@11.0.6/java.nio.channels.Channels.writeFullyImpl(Channels.java:74)
java.base@11.0.6/java.nio.channels.Channels.writeFully(Channels.java:97)
java.base@11.0.6/java.nio.channels.Channels$1.write(Channels.java:172)
app//org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:416)
java.base@11.0.6/java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:74)
java.base@11.0.6/java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81)
java.base@11.0.6/java.io.BufferedOutputStream.write(BufferedOutputStream.java:127)
app//org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53)
app//org.elasticsearch.common.lucene.store.FilterIndexOutput.writeBytes(FilterIndexOutput.java:59)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:73)
app//org.apache.lucene.util.bkd.BKDWriter$OneDimensionBKDWriter.writeLeafBlock(BKDWriter.java:666)
app//org.apache.lucene.util.bkd.BKDWriter$OneDimensionBKDWriter.add(BKDWriter.java:586)
app//org.apache.lucene.util.bkd.BKDWriter.merge(BKDWriter.java:515)
app//org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:213)
app//org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:202)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:162)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
3/10 snapshots sharing following 9 elements
app//org.apache.lucene.util.bkd.BKDWriter.merge(BKDWriter.java:517)
app//org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:213)
app//org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:202)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:162)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
3/10 snapshots sharing following 8 elements
app//org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:213)
app//org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:202)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:162)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
75.1% (375.7ms out of 500ms) cpu usage by thread 'elasticsearch[a253de2f08f5d6b1348f293c60376d38][refresh][T#2]'
2/10 snapshots sharing following 38 elements
app//org.apache.lucene.codecs.lucene80.IndexedDISI.advance(IndexedDISI.java:384)
app//org.apache.lucene.codecs.lucene80.IndexedDISI.nextDoc(IndexedDISI.java:459)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$SparseNumericDocValues.nextDoc(Lucene80DocValuesProducer.java:444)
app//org.apache.lucene.index.ReadersAndUpdates$MergedDocValues.nextDoc(ReadersAndUpdates.java:469)
app//org.apache.lucene.index.ReadersAndUpdates$2$1.nextDoc(ReadersAndUpdates.java:396)
app//org.apache.lucene.index.SingletonSortedNumericDocValues.nextDoc(SingletonSortedNumericDocValues.java:53)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:168)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373)
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587)
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230)
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
2/10 snapshots sharing following 35 elements
java.base@11.0.6/java.util.HashMap.hash(HashMap.java:339)
java.base@11.0.6/java.util.HashMap.put(HashMap.java:607)
java.base@11.0.6/java.util.HashSet.add(HashSet.java:220)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:190)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373)
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587)
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230)
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
3/10 snapshots sharing following 35 elements
app//org.apache.lucene.index.ReadersAndUpdates$2$1.nextDoc(ReadersAndUpdates.java:396)
app//org.apache.lucene.index.SingletonSortedNumericDocValues.nextDoc(SingletonSortedNumericDocValues.java:53)
app//org.apache.lucene.codecs.lucene80.IndexedDISI.writeBitSet(IndexedDISI.java:196)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:221)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373)
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587)
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230)
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
2/10 snapshots sharing following 49 elements
java.base@11.0.6/sun.nio.ch.NativeThread.current(Native Method)
java.base@11.0.6/sun.nio.ch.NativeThreadSet.add(NativeThreadSet.java:46)
java.base@11.0.6/sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:276)
java.base@11.0.6/java.nio.channels.Channels.writeFullyImpl(Channels.java:74)
java.base@11.0.6/java.nio.channels.Channels.writeFully(Channels.java:97)
java.base@11.0.6/java.nio.channels.Channels$1.write(Channels.java:172)
app//org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:416)
java.base@11.0.6/java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:74)
java.base@11.0.6/java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81)
java.base@11.0.6/java.io.BufferedOutputStream.write(BufferedOutputStream.java:95)
app//org.apache.lucene.store.OutputStreamIndexOutput.writeByte(OutputStreamIndexOutput.java:47)
app//org.elasticsearch.common.lucene.store.FilterIndexOutput.writeByte(FilterIndexOutput.java:54)
app//org.elasticsearch.index.store.ByteSizeCachingDirectory$2.writeByte(ByteSizeCachingDirectory.java:154)
app//org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:70)
app//org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:212)
app//org.apache.lucene.codecs.lucene80.IndexedDISI.flush(IndexedDISI.java:121)
app//org.apache.lucene.codecs.lucene80.IndexedDISI.writeBitSet(IndexedDISI.java:203)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:221)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373)
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587)
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230)
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
unique snapshot
app//org.apache.lucene.codecs.lucene80.IndexedDISI.writeBitSet(IndexedDISI.java:203)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:221)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373)
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587)
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230)
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
::: {741248536f8a624a46358affbf3940d9}{lunuJX_4QM-QSGxUuAK6aQ}{K6N5Dqy9TUOjP5v5nWfonw}{x.x.x.x}{x.x.x.x:9300}{dir}{distributed_snapshot_deletion_enabled=true, zone=us-east-1d, cross_cluster_transport_address=x.x.x.x}
Hot threads at 2021-01-15T01:04:30.188Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
100.2% (500.8ms out of 500ms) cpu usage by thread 'elasticsearch[741248536f8a624a46358affbf3940d9][[people][3]: Lucene Merge Thread #3804]'
2/10 snapshots sharing following 13 elements
app//org.apache.lucene.util.compress.LZ4.encodeSequence(LZ4.java:177)
app//org.apache.lucene.util.compress.LZ4.compress(LZ4.java:441)
app//org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:165)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:229)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:159)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:518)
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:229)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
2/10 snapshots sharing following 13 elements
app//org.apache.lucene.util.compress.LZ4.decompress(LZ4.java:103)
app//org.apache.lucene.codecs.compressing.CompressionMode$4.decompress(CompressionMode.java:139)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader$BlockState.doReset(CompressingStoredFieldsReader.java:480)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader$BlockState.reset(CompressingStoredFieldsReader.java:393)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.document(CompressingStoredFieldsReader.java:572)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:514)
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:229)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
2/10 snapshots sharing following 14 elements
app//org.apache.lucene.util.compress.LZ4.encodeLiterals(LZ4.java:151)
app//org.apache.lucene.util.compress.LZ4.encodeSequence(LZ4.java:172)
app//org.apache.lucene.util.compress.LZ4.compress(LZ4.java:441)
app//org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:165)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:229)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:159)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:518)
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:229)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
3/10 snapshots sharing following 17 elements
app//org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53)
app//org.elasticsearch.common.lucene.store.FilterIndexOutput.writeBytes(FilterIndexOutput.java:59)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:73)
app//org.apache.lucene.util.compress.LZ4.encodeLiterals(LZ4.java:159)
app//org.apache.lucene.util.compress.LZ4.encodeSequence(LZ4.java:172)
app//org.apache.lucene.util.compress.LZ4.compress(LZ4.java:441)
app//org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:165)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:229)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:159)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:518)
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:229)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
unique snapshot
app//org.apache.lucene.util.compress.LZ4.decompress(LZ4.java:112)
app//org.apache.lucene.codecs.compressing.CompressionMode$4.decompress(CompressionMode.java:139)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader$BlockState.doReset(CompressingStoredFieldsReader.java:480)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader$BlockState.reset(CompressingStoredFieldsReader.java:393)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.document(CompressingStoredFieldsReader.java:572)
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:514)
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:229)
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
64.1% (320.4ms out of 500ms) cpu usage by thread 'elasticsearch[741248536f8a624a46358affbf3940d9][refresh][T#1]'
2/10 snapshots sharing following 35 elements
app//org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:212)
app//org.apache.lucene.codecs.lucene80.IndexedDISI.flush(IndexedDISI.java:121)
app//org.apache.lucene.codecs.lucene80.IndexedDISI.writeBitSet(IndexedDISI.java:203)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:221)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373)
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587)
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230)
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
3/10 snapshots sharing following 39 elements
app//org.apache.lucene.codecs.lucene80.IndexedDISI.advance(IndexedDISI.java:384)
app//org.apache.lucene.codecs.lucene80.IndexedDISI.nextDoc(IndexedDISI.java:459)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$SparseNumericDocValues.nextDoc(Lucene80DocValuesProducer.java:444)
app//org.apache.lucene.index.ReadersAndUpdates$MergedDocValues.nextDoc(ReadersAndUpdates.java:469)
app//org.apache.lucene.index.ReadersAndUpdates$2$1.nextDoc(ReadersAndUpdates.java:396)
app//org.apache.lucene.index.SingletonSortedNumericDocValues.nextDoc(SingletonSortedNumericDocValues.java:53)
app//org.apache.lucene.codecs.lucene80.IndexedDISI.writeBitSet(IndexedDISI.java:196)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:221)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373)
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587)
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230)
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
2/10 snapshots sharing following 34 elements
app//org.apache.lucene.index.SingletonSortedNumericDocValues.nextDoc(SingletonSortedNumericDocValues.java:53)
app//org.apache.lucene.codecs.lucene80.IndexedDISI.writeBitSet(IndexedDISI.java:196)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:221)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373)
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587)
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230)
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
2/10 snapshots sharing following 33 elements
app//org.apache.lucene.codecs.lucene80.IndexedDISI.writeBitSet(IndexedDISI.java:203)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:221)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373)
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587)
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230)
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
unique snapshot
app//org.apache.lucene.codecs.lucene80.IndexedDISI.writeBitSet(IndexedDISI.java:196)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:221)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118)
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109)
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373)
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587)
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230)
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311)
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362)
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625)
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604)
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304)
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840)
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111)
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972)
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
0.7% (3.3ms out of 500ms) cpu usage by thread 'elasticsearch[741248536f8a624a46358affbf3940d9][[people][3]: Lucene Merge Thread #3794]'
10/10 snapshots sharing following 16 elements
java.base@11.0.6/jdk.internal.misc.Unsafe.park(Native Method)
java.base@11.0.6/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
java.base@11.0.6/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:156)
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:96)
app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5014)
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4527)
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment