Created
January 15, 2021 01:08
-
-
Save rex-remind101/b6b1a8d1b82d601d343203b0ea44add2 to your computer and use it in GitHub Desktop.
hot_threads ES
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Attempt 1 | |
[hadoop@ip-10-128-4-195 ~]$ curl <host>/_nodes/hot_threads | |
::: {741248536f8a624a46358affbf3940d9}{lunuJX_4QM-QSGxUuAK6aQ}{K6N5Dqy9TUOjP5v5nWfonw}{x.x.x.x}{x.x.x.x:9300}{dir}{distributed_snapshot_deletion_enabled=true, zone=us-east-1d, cross_cluster_transport_address=x.x.x.x, } | |
Hot threads at 2021-01-15T00:58:24.674Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
76.1% (380.4ms out of 500ms) cpu usage by thread 'elasticsearch[741248536f8a624a46358affbf3940d9][refresh][T#2]' | |
5/10 snapshots sharing following 36 elements | |
app//org.apache.lucene.index.PointValuesWriter$1.intersect(PointValuesWriter.java:89) | |
app//org.apache.lucene.index.PointValuesWriter$MutableSortingPointValues.intersect(PointValuesWriter.java:203) | |
app//org.apache.lucene.util.bkd.BKDWriter.writeField1Dim(BKDWriter.java:466) | |
app//org.apache.lucene.util.bkd.BKDWriter.writeField(BKDWriter.java:384) | |
app//org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.writeField(Lucene60PointsWriter.java:104) | |
app//org.apache.lucene.index.PointValuesWriter.flush(PointValuesWriter.java:188) | |
app//org.apache.lucene.index.DefaultIndexingChain.writePoints(DefaultIndexingChain.java:218) | |
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:144) | |
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468) | |
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555) | |
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722) | |
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262) | |
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112) | |
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225) | |
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625) | |
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604) | |
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304) | |
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840) | |
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111) | |
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972) | |
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144) | |
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
5/10 snapshots sharing following 37 elements | |
app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:128) | |
app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:121) | |
app//org.apache.lucene.util.bkd.MutablePointsReaderUtils.sort(MutablePointsReaderUtils.java:91) | |
app//org.apache.lucene.util.bkd.BKDWriter.writeField1Dim(BKDWriter.java:462) | |
app//org.apache.lucene.util.bkd.BKDWriter.writeField(BKDWriter.java:384) | |
app//org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.writeField(Lucene60PointsWriter.java:104) | |
app//org.apache.lucene.index.PointValuesWriter.flush(PointValuesWriter.java:188) | |
app//org.apache.lucene.index.DefaultIndexingChain.writePoints(DefaultIndexingChain.java:218) | |
app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:144) | |
app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468) | |
app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555) | |
app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722) | |
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262) | |
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112) | |
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225) | |
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625) | |
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604) | |
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304) | |
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840) | |
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111) | |
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972) | |
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144) | |
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
0.0% (103.7micros out of 500ms) cpu usage by thread 'MetricAggregationClientPublisher-1' | |
10/10 snapshots sharing following 13 elements | |
java.base@11.0.6/jdk.internal.misc.Unsafe.park(Native Method) | |
java.base@11.0.6/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) | |
java.base@11.0.6/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) | |
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:513) | |
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:675) | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
java.base@11.0.6/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) | |
java.base@11.0.6/java.util.concurrent.FutureTask.run(FutureTask.java:264) | |
java.base@11.0.6/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
::: {428abb0d265d296a6b15e382621d60fd}{ToM_ylKTSVejdIAfH_O5DA}{e_ITHIwCTCGaBJVmTmn-YQ}{x.x.x.x}{x.x.x.x:9300}{dir}{distributed_snapshot_deletion_enabled=true, zone=us-east-1b, cross_cluster_transport_address=x.x.x.x} | |
Hot threads at 2021-01-15T00:58:24.675Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
27.3% (136.6ms out of 500ms) cpu usage by thread 'qtp1356445494-244187' | |
4/10 snapshots sharing following 47 elements | |
app//com.fasterxml.jackson.core.json.UTF8JsonGenerator.writeNumber(UTF8JsonGenerator.java:958) | |
app//com.fasterxml.jackson.core.JsonGenerator._copyCurrentContents(JsonGenerator.java:1975) | |
app//com.fasterxml.jackson.core.JsonGenerator.copyCurrentStructure(JsonGenerator.java:1914) | |
app//org.elasticsearch.common.xcontent.json.JsonXContentGenerator.copyCurrentStructure(JsonXContentGenerator.java:418) | |
app//org.elasticsearch.common.xcontent.XContentBuilder.copyCurrentStructure(XContentBuilder.java:997) | |
app//org.elasticsearch.action.update.UpdateRequest.lambda$static$3(UpdateRequest.java:82) | |
app//org.elasticsearch.action.update.UpdateRequest$$Lambda$3431/0x000000080177a040.parse(Unknown Source) | |
app//org.elasticsearch.common.xcontent.AbstractObjectParser.lambda$declareObject$1(AbstractObjectParser.java:169) | |
app//org.elasticsearch.common.xcontent.AbstractObjectParser$$Lambda$298/0x00000008002c4040.parse(Unknown Source) | |
app//org.elasticsearch.common.xcontent.ObjectParser.lambda$declareField$9(ObjectParser.java:386) | |
app//org.elasticsearch.common.xcontent.ObjectParser$$Lambda$286/0x00000008002c1040.parse(Unknown Source) | |
app//org.elasticsearch.common.xcontent.ObjectParser.parseValue(ObjectParser.java:529) | |
app//org.elasticsearch.common.xcontent.ObjectParser.parseSub(ObjectParser.java:541) | |
app//org.elasticsearch.common.xcontent.ObjectParser.parse(ObjectParser.java:324) | |
app//org.elasticsearch.action.update.UpdateRequest.fromXContent(UpdateRequest.java:851) | |
app//org.elasticsearch.action.bulk.BulkRequestParser.parse(BulkRequestParser.java:290) | |
app//org.elasticsearch.action.bulk.BulkRequest.add(BulkRequest.java:304) | |
app//org.elasticsearch.rest.action.document.RestBulkAction.prepareRequest(RestBulkAction.java:101) | |
app//org.elasticsearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:94) | |
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:236) | |
app//org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:318) | |
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:176) | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301) | |
app//[AMAZON INTERNAL] | |
app//org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) | |
app//[AMAZON INTERNAL] | |
app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) | |
app//org.eclipse.jetty.server.Server.handle(Server.java:370) | |
app//org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489) | |
app//org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:960) | |
app//org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1021) | |
app//org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865) | |
app//org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) | |
app//org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) | |
app//org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668) | |
app//org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52) | |
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) | |
app//org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
2/10 snapshots sharing following 48 elements | |
app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser._nextTokenNotInObject(UTF8StreamJsonParser.java:855) | |
app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextToken(UTF8StreamJsonParser.java:754) | |
app//com.fasterxml.jackson.core.JsonGenerator._copyCurrentContents(JsonGenerator.java:1935) | |
app//com.fasterxml.jackson.core.JsonGenerator.copyCurrentStructure(JsonGenerator.java:1914) | |
app//org.elasticsearch.common.xcontent.json.JsonXContentGenerator.copyCurrentStructure(JsonXContentGenerator.java:418) | |
app//org.elasticsearch.common.xcontent.XContentBuilder.copyCurrentStructure(XContentBuilder.java:997) | |
app//org.elasticsearch.action.update.UpdateRequest.lambda$static$5(UpdateRequest.java:88) | |
app//org.elasticsearch.action.update.UpdateRequest$$Lambda$3433/0x000000080177a840.parse(Unknown Source) | |
app//org.elasticsearch.common.xcontent.AbstractObjectParser.lambda$declareObject$1(AbstractObjectParser.java:169) | |
app//org.elasticsearch.common.xcontent.AbstractObjectParser$$Lambda$298/0x00000008002c4040.parse(Unknown Source) | |
app//org.elasticsearch.common.xcontent.ObjectParser.lambda$declareField$9(ObjectParser.java:386) | |
app//org.elasticsearch.common.xcontent.ObjectParser$$Lambda$286/0x00000008002c1040.parse(Unknown Source) | |
app//org.elasticsearch.common.xcontent.ObjectParser.parseValue(ObjectParser.java:529) | |
app//org.elasticsearch.common.xcontent.ObjectParser.parseSub(ObjectParser.java:541) | |
app//org.elasticsearch.common.xcontent.ObjectParser.parse(ObjectParser.java:324) | |
app//org.elasticsearch.action.update.UpdateRequest.fromXContent(UpdateRequest.java:851) | |
app//org.elasticsearch.action.bulk.BulkRequestParser.parse(BulkRequestParser.java:290) | |
app//org.elasticsearch.action.bulk.BulkRequest.add(BulkRequest.java:304) | |
app//org.elasticsearch.rest.action.document.RestBulkAction.prepareRequest(RestBulkAction.java:101) | |
app//org.elasticsearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:94) | |
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:236) | |
app//org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:318) | |
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:176) | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301) | |
app//[AMAZON INTERNAL] | |
app//org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) | |
app//[AMAZON INTERNAL] | |
app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) | |
app//org.eclipse.jetty.server.Server.handle(Server.java:370) | |
app//org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489) | |
app//org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:960) | |
app//org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1021) | |
app//org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865) | |
app//org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) | |
app//org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) | |
app//org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668) | |
app//org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52) | |
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) | |
app//org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
2/10 snapshots sharing following 45 elements | |
app//com.fasterxml.jackson.core.JsonGenerator.copyCurrentStructure(JsonGenerator.java:1914) | |
app//org.elasticsearch.common.xcontent.json.JsonXContentGenerator.copyCurrentStructure(JsonXContentGenerator.java:418) | |
app//org.elasticsearch.common.xcontent.XContentBuilder.copyCurrentStructure(XContentBuilder.java:997) | |
app//org.elasticsearch.action.update.UpdateRequest.lambda$static$3(UpdateRequest.java:82) | |
app//org.elasticsearch.action.update.UpdateRequest$$Lambda$3431/0x000000080177a040.parse(Unknown Source) | |
app//org.elasticsearch.common.xcontent.AbstractObjectParser.lambda$declareObject$1(AbstractObjectParser.java:169) | |
app//org.elasticsearch.common.xcontent.AbstractObjectParser$$Lambda$298/0x00000008002c4040.parse(Unknown Source) | |
app//org.elasticsearch.common.xcontent.ObjectParser.lambda$declareField$9(ObjectParser.java:386) | |
app//org.elasticsearch.common.xcontent.ObjectParser$$Lambda$286/0x00000008002c1040.parse(Unknown Source) | |
app//org.elasticsearch.common.xcontent.ObjectParser.parseValue(ObjectParser.java:529) | |
app//org.elasticsearch.common.xcontent.ObjectParser.parseSub(ObjectParser.java:541) | |
app//org.elasticsearch.common.xcontent.ObjectParser.parse(ObjectParser.java:324) | |
app//org.elasticsearch.action.update.UpdateRequest.fromXContent(UpdateRequest.java:851) | |
app//org.elasticsearch.action.bulk.BulkRequestParser.parse(BulkRequestParser.java:290) | |
app//org.elasticsearch.action.bulk.BulkRequest.add(BulkRequest.java:304) | |
app//org.elasticsearch.rest.action.document.RestBulkAction.prepareRequest(RestBulkAction.java:101) | |
app//org.elasticsearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:94) | |
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:236) | |
app//org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:318) | |
app//org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:176) | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//[AMAZON INTERNAL] | |
app//org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301) | |
app//[AMAZON INTERNAL] | |
app//org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) | |
app//[AMAZON INTERNAL] | |
app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) | |
app//org.eclipse.jetty.server.Server.handle(Server.java:370) | |
app//org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489) | |
app//org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:960) | |
app//org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1021) | |
app//org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865) | |
app//org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) | |
app//org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) | |
app//org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668) | |
app//org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52) | |
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) | |
app//org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
2/10 snapshots sharing following 1 elements | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
::: {c9866516a52bdf0eb71c2ed62b04fa13}{X5AJ7ng4SZq9ZwCHRm4TyA}{8eWWFnuPQ3yIaCvSvBW7LA}{x.x.x.x}{x.x.x.x:9300}{mr}{distributed_snapshot_deletion_enabled=true, zone=us-east-1d, cross_cluster_transport_address=x.x.x.x} | |
Hot threads at 2021-01-15T00:58:24.675Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
0.0% (92.5micros out of 500ms) cpu usage by thread 'MetricAggregationClientPublisher-1' | |
10/10 snapshots sharing following 13 elements | |
java.base@11.0.6/jdk.internal.misc.Unsafe.park(Native Method) | |
java.base@11.0.6/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) | |
java.base@11.0.6/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) | |
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:513) | |
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:675) | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
java.base@11.0.6/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) | |
java.base@11.0.6/java.util.concurrent.FutureTask.run(FutureTask.java:264) | |
java.base@11.0.6/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
::: {a253de2f08f5d6b1348f293c60376d38}{yJuxMUq4QN-es3z064UIEw}{hmZe3HskR5GaZ8eoPGMS4w}{x.x.x.x}{x.x.x.x:9300}{dir}{distributed_snapshot_deletion_enabled=true, zone=us-east-1a, cross_cluster_transport_address=x.x.x.x} | |
Hot threads at 2021-01-15T00:58:24.675Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
0.1% (251.4micros out of 500ms) cpu usage by thread 'MetricAggregationClientPublisher-1' | |
10/10 snapshots sharing following 13 elements | |
java.base@11.0.6/jdk.internal.misc.Unsafe.park(Native Method) | |
java.base@11.0.6/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) | |
java.base@11.0.6/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) | |
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:513) | |
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:675) | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
java.base@11.0.6/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) | |
java.base@11.0.6/java.util.concurrent.FutureTask.run(FutureTask.java:264) | |
java.base@11.0.6/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
::: {c1715d6e1bcc3d0e71d0c8a5bc0082ef}{BuCONqzKT8yNJfaebdT_CA}{hatq8-wvSe2dOG8Ynttx8A}{x.x.x.x}{x.x.x.x:9300}{mr}{distributed_snapshot_deletion_enabled=true, zone=us-east-1a, cross_cluster_transport_address=x.x.x.x} | |
Hot threads at 2021-01-15T00:58:24.690Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
0.0% (90.4micros out of 500ms) cpu usage by thread 'MetricAggregationClientPublisher-1' | |
10/10 snapshots sharing following 13 elements | |
java.base@11.0.6/jdk.internal.misc.Unsafe.park(Native Method) | |
java.base@11.0.6/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) | |
java.base@11.0.6/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) | |
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:513) | |
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:675) | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
java.base@11.0.6/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) | |
java.base@11.0.6/java.util.concurrent.FutureTask.run(FutureTask.java:264) | |
java.base@11.0.6/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
::: {3b82871f297f3450044d713d34411d6e}{ftE1pCbqR96KoPMpx2Tl-A}{InjOcO1UTaCVsJZFpLdaow}{x.x.x.x}{x.x.x.x:9300}{mr}{distributed_snapshot_deletion_enabled=true, zone=us-east-1b, cross_cluster_transport_address=x.x.x.x} | |
Hot threads at 2021-01-15T00:58:24.675Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
1.1% (5.5ms out of 500ms) cpu usage by thread 'elasticsearch[3b82871f297f3450044d713d34411d6e][generic][T#192]' | |
10/10 snapshots sharing following 55 elements | |
app//org.conscrypt.NativeCrypto.SSL_read(Native Method) | |
app//org.conscrypt.NativeSsl.read(NativeSsl.java:409) | |
app//org.conscrypt.ConscryptFileDescriptorSocket$SSLInputStream.read(ConscryptFileDescriptorSocket.java:548) | |
app//org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137) | |
app//org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153) | |
app//org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280) | |
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138) | |
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56) | |
app//org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259) | |
app//org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163) | |
org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157) | |
app//org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273) | |
[AMAZON INTERNAL] | |
app//org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125) | |
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) | |
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) | |
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) | |
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) | |
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56) | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
org.elasticsearch.repositories.s3.S3RetryingInputStream.lambda$openStream$0(S3RetryingInputStream.java:97) | |
org.elasticsearch.repositories.s3.S3RetryingInputStream$$Lambda$3555/0x000000010082f440.run(Unknown Source) | |
java.base@11.0.6/java.security.AccessController.doPrivileged(Native Method) | |
org.elasticsearch.repositories.s3.SocketAccess.doPrivileged(SocketAccess.java:42) | |
org.elasticsearch.repositories.s3.S3RetryingInputStream.openStream(S3RetryingInputStream.java:97) | |
org.elasticsearch.repositories.s3.S3RetryingInputStream.<init>(S3RetryingInputStream.java:85) | |
org.elasticsearch.repositories.s3.S3RetryingInputStream.<init>(S3RetryingInputStream.java:68) | |
org.elasticsearch.repositories.s3.S3BlobContainer.readBlob(S3BlobContainer.java:99) | |
app//org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat.readBlob(ChecksumBlobStoreFormat.java:150) | |
app//org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat.read(ChecksumBlobStoreFormat.java:136) | |
app//org.elasticsearch.repositories.blobstore.BlobStoreRepository.getSnapshotInfo(BlobStoreRepository.java:1308) | |
app//org.elasticsearch.action.admin.cluster.snapshots.get.TransportGetSnapshotsAction.snapshots(TransportGetSnapshotsAction.java:212) | |
app//org.elasticsearch.action.admin.cluster.snapshots.get.TransportGetSnapshotsAction.masterOperation(TransportGetSnapshotsAction.java:151) | |
app//org.elasticsearch.action.admin.cluster.snapshots.get.TransportGetSnapshotsAction.masterOperation(TransportGetSnapshotsAction.java:67) | |
app//org.elasticsearch.action.support.master.TransportMasterNodeAction.masterOperation(TransportMasterNodeAction.java:100) | |
app//org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.lambda$doStart$3(TransportMasterNodeAction.java:171) | |
app//org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$$Lambda$3216/0x0000000101704440.accept(Unknown Source) | |
app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73) | |
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:704) | |
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
# Attempt 2 | |
curl <host>/_nodes/hot_threads | |
::: {c9866516a52bdf0eb71c2ed62b04fa13}{X5AJ7ng4SZq9ZwCHRm4TyA}{8eWWFnuPQ3yIaCvSvBW7LA}{x.x.x.x}{x.x.x.x:9300}{mr}{distributed_snapshot_deletion_enabled=true, zone=us-east-1d, cross_cluster_transport_address=x.x.x.x} | |
Hot threads at 2021-01-15T01:04:30.188Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
0.0% (98.6micros out of 500ms) cpu usage by thread 'MetricAggregationClientPublisher-1' | |
10/10 snapshots sharing following 13 elements | |
java.base@11.0.6/jdk.internal.misc.Unsafe.park(Native Method) | |
java.base@11.0.6/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) | |
java.base@11.0.6/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) | |
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:513) | |
java.base@11.0.6/java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:675) | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
java.base@11.0.6/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) | |
java.base@11.0.6/java.util.concurrent.FutureTask.run(FutureTask.java:264) | |
java.base@11.0.6/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
::: {428abb0d265d296a6b15e382621d60fd}{ToM_ylKTSVejdIAfH_O5DA}{e_ITHIwCTCGaBJVmTmn-YQ}{x.x.x.x}{x.x.x.x:9300}{dir}{distributed_snapshot_deletion_enabled=true, zone=us-east-1b, cross_cluster_transport_address=x.x.x.x, } | |
Hot threads at 2021-01-15T01:04:30.187Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
::: {3b82871f297f3450044d713d34411d6e}{ftE1pCbqR96KoPMpx2Tl-A}{InjOcO1UTaCVsJZFpLdaow}{x.x.x.x}{x.x.x.x:9300}{mr}{distributed_snapshot_deletion_enabled=true, zone=us-east-1b, cross_cluster_transport_address=x.x.x.x} | |
Hot threads at 2021-01-15T01:04:30.187Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
::: {c1715d6e1bcc3d0e71d0c8a5bc0082ef}{BuCONqzKT8yNJfaebdT_CA}{hatq8-wvSe2dOG8Ynttx8A}{x.x.x.x}{x.x.x.x:9300}{mr}{distributed_snapshot_deletion_enabled=true, zone=us-east-1a, cross_cluster_transport_address=x.x.x.x} | |
Hot threads at 2021-01-15T01:04:30.187Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
::: {a253de2f08f5d6b1348f293c60376d38}{yJuxMUq4QN-es3z064UIEw}{hmZe3HskR5GaZ8eoPGMS4w}{x.x.x.x}{x.x.x.x:9300}{dir}{distributed_snapshot_deletion_enabled=true, zone=us-east-1a, cross_cluster_transport_address=x.x.x.x} | |
Hot threads at 2021-01-15T01:04:30.188Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
89.9% (449.3ms out of 500ms) cpu usage by thread 'elasticsearch[a253de2f08f5d6b1348f293c60376d38][write][T#1]' | |
2/10 snapshots sharing following 40 elements | |
app//org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:71) | |
app//org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:267) | |
app//org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:841) | |
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:818) | |
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:790) | |
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:267) | |
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:157) | |
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:202) | |
app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:114) | |
app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:81) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:932) | |
app//org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:109) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.runWithPrimaryShardReference(TransportReplicationAction.java:397) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.lambda$doRun$0(TransportReplicationAction.java:320) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction$$Lambda$3945/0x0000000801a3ac40.accept(Unknown Source) | |
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) | |
app//org.elasticsearch.index.shard.IndexShard.lambda$wrapPrimaryOperationPermitListener$24(IndexShard.java:2860) | |
app//org.elasticsearch.index.shard.IndexShard$$Lambda$3947/0x0000000801a3b440.accept(Unknown Source) | |
app//org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:113) | |
app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:285) | |
app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:237) | |
app//org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationPermit(IndexShard.java:2834) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryOperationPermit(TransportReplicationAction.java:873) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:316) | |
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction.handlePrimaryRequest(TransportReplicationAction.java:279) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$$Lambda$2445/0x0000000800d12c40.messageReceived(Unknown Source) | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) | |
app//org.elasticsearch.transport.TransportService$8.doRun(TransportService.java:801) | |
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44) | |
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:704) | |
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
4/10 snapshots sharing following 46 elements | |
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:406) | |
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:250) | |
app//org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:495) | |
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1594) | |
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213) | |
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1171) | |
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1108) | |
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:948) | |
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:861) | |
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:833) | |
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:790) | |
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:267) | |
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:157) | |
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:202) | |
app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:114) | |
app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:81) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:932) | |
app//org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:109) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.runWithPrimaryShardReference(TransportReplicationAction.java:397) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.lambda$doRun$0(TransportReplicationAction.java:320) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction$$Lambda$3945/0x0000000801a3ac40.accept(Unknown Source) | |
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) | |
app//org.elasticsearch.index.shard.IndexShard.lambda$wrapPrimaryOperationPermitListener$24(IndexShard.java:2860) | |
app//org.elasticsearch.index.shard.IndexShard$$Lambda$3947/0x0000000801a3b440.accept(Unknown Source) | |
app//org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:113) | |
app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:285) | |
app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:237) | |
app//org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationPermit(IndexShard.java:2834) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryOperationPermit(TransportReplicationAction.java:873) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:316) | |
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction.handlePrimaryRequest(TransportReplicationAction.java:279) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$$Lambda$2445/0x0000000800d12c40.messageReceived(Unknown Source) | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) | |
app//org.elasticsearch.transport.TransportService$8.doRun(TransportService.java:801) | |
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44) | |
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:704) | |
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
4/10 snapshots sharing following 52 elements | |
app//org.apache.lucene.util.compress.LZ4.compress(LZ4.java:441) | |
app//org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:165) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:229) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:159) | |
app//org.apache.lucene.index.StoredFieldsConsumer.finishDocument(StoredFieldsConsumer.java:59) | |
app//org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:378) | |
app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:414) | |
app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:250) | |
app//org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:495) | |
app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1594) | |
app//org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213) | |
app//org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1171) | |
app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1108) | |
app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:948) | |
app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:861) | |
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:833) | |
app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:790) | |
app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:267) | |
app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:157) | |
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:202) | |
app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:114) | |
app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:81) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:932) | |
app//org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:109) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.runWithPrimaryShardReference(TransportReplicationAction.java:397) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.lambda$doRun$0(TransportReplicationAction.java:320) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction$$Lambda$3945/0x0000000801a3ac40.accept(Unknown Source) | |
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) | |
app//org.elasticsearch.index.shard.IndexShard.lambda$wrapPrimaryOperationPermitListener$24(IndexShard.java:2860) | |
app//org.elasticsearch.index.shard.IndexShard$$Lambda$3947/0x0000000801a3b440.accept(Unknown Source) | |
app//org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:113) | |
app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:285) | |
app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:237) | |
app//org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationPermit(IndexShard.java:2834) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryOperationPermit(TransportReplicationAction.java:873) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:316) | |
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction.handlePrimaryRequest(TransportReplicationAction.java:279) | |
app//org.elasticsearch.action.support.replication.TransportReplicationAction$$Lambda$2445/0x0000000800d12c40.messageReceived(Unknown Source) | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
[AMAZON INTERNAL] | |
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) | |
app//org.elasticsearch.transport.TransportService$8.doRun(TransportService.java:801) | |
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44) | |
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:704) | |
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
82.9% (414.3ms out of 500ms) cpu usage by thread 'elasticsearch[a253de2f08f5d6b1348f293c60376d38][[people][3]: Lucene Merge Thread #17542]' | |
2/10 snapshots sharing following 11 elements | |
app//org.apache.lucene.util.PriorityQueue.downHeap(PriorityQueue.java:267) | |
app//org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:202) | |
app//org.apache.lucene.util.bkd.BKDWriter.merge(BKDWriter.java:518) | |
app//org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:213) | |
app//org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:202) | |
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:162) | |
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463) | |
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625) | |
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662) | |
2/10 snapshots sharing following 22 elements | |
java.base@11.0.6/sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:280) | |
java.base@11.0.6/java.nio.channels.Channels.writeFullyImpl(Channels.java:74) | |
java.base@11.0.6/java.nio.channels.Channels.writeFully(Channels.java:97) | |
java.base@11.0.6/java.nio.channels.Channels$1.write(Channels.java:172) | |
app//org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:416) | |
java.base@11.0.6/java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:74) | |
java.base@11.0.6/java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) | |
java.base@11.0.6/java.io.BufferedOutputStream.write(BufferedOutputStream.java:127) | |
app//org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
app//org.elasticsearch.common.lucene.store.FilterIndexOutput.writeBytes(FilterIndexOutput.java:59) | |
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:73) | |
app//org.apache.lucene.util.bkd.BKDWriter$OneDimensionBKDWriter.writeLeafBlock(BKDWriter.java:666) | |
app//org.apache.lucene.util.bkd.BKDWriter$OneDimensionBKDWriter.add(BKDWriter.java:586) | |
app//org.apache.lucene.util.bkd.BKDWriter.merge(BKDWriter.java:515) | |
app//org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:213) | |
app//org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:202) | |
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:162) | |
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463) | |
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625) | |
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662) | |
3/10 snapshots sharing following 9 elements | |
app//org.apache.lucene.util.bkd.BKDWriter.merge(BKDWriter.java:517) | |
app//org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:213) | |
app//org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:202) | |
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:162) | |
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463) | |
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625) | |
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662) | |
3/10 snapshots sharing following 8 elements | |
app//org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:213) | |
app//org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:202) | |
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:162) | |
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463) | |
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625) | |
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662) | |
75.1% (375.7ms out of 500ms) cpu usage by thread 'elasticsearch[a253de2f08f5d6b1348f293c60376d38][refresh][T#2]' | |
2/10 snapshots sharing following 38 elements | |
app//org.apache.lucene.codecs.lucene80.IndexedDISI.advance(IndexedDISI.java:384) | |
app//org.apache.lucene.codecs.lucene80.IndexedDISI.nextDoc(IndexedDISI.java:459) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$SparseNumericDocValues.nextDoc(Lucene80DocValuesProducer.java:444) | |
app//org.apache.lucene.index.ReadersAndUpdates$MergedDocValues.nextDoc(ReadersAndUpdates.java:469) | |
app//org.apache.lucene.index.ReadersAndUpdates$2$1.nextDoc(ReadersAndUpdates.java:396) | |
app//org.apache.lucene.index.SingletonSortedNumericDocValues.nextDoc(SingletonSortedNumericDocValues.java:53) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:168) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118) | |
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109) | |
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373) | |
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587) | |
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230) | |
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311) | |
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262) | |
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112) | |
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225) | |
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625) | |
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604) | |
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304) | |
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840) | |
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111) | |
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972) | |
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144) | |
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
2/10 snapshots sharing following 35 elements | |
java.base@11.0.6/java.util.HashMap.hash(HashMap.java:339) | |
java.base@11.0.6/java.util.HashMap.put(HashMap.java:607) | |
java.base@11.0.6/java.util.HashSet.add(HashSet.java:220) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:190) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118) | |
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109) | |
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373) | |
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587) | |
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230) | |
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311) | |
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262) | |
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112) | |
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225) | |
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625) | |
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604) | |
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304) | |
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840) | |
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111) | |
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972) | |
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144) | |
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
3/10 snapshots sharing following 35 elements | |
app//org.apache.lucene.index.ReadersAndUpdates$2$1.nextDoc(ReadersAndUpdates.java:396) | |
app//org.apache.lucene.index.SingletonSortedNumericDocValues.nextDoc(SingletonSortedNumericDocValues.java:53) | |
app//org.apache.lucene.codecs.lucene80.IndexedDISI.writeBitSet(IndexedDISI.java:196) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:221) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118) | |
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109) | |
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373) | |
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587) | |
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230) | |
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311) | |
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262) | |
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112) | |
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225) | |
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625) | |
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604) | |
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304) | |
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840) | |
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111) | |
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972) | |
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144) | |
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
2/10 snapshots sharing following 49 elements | |
java.base@11.0.6/sun.nio.ch.NativeThread.current(Native Method) | |
java.base@11.0.6/sun.nio.ch.NativeThreadSet.add(NativeThreadSet.java:46) | |
java.base@11.0.6/sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:276) | |
java.base@11.0.6/java.nio.channels.Channels.writeFullyImpl(Channels.java:74) | |
java.base@11.0.6/java.nio.channels.Channels.writeFully(Channels.java:97) | |
java.base@11.0.6/java.nio.channels.Channels$1.write(Channels.java:172) | |
app//org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:416) | |
java.base@11.0.6/java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:74) | |
java.base@11.0.6/java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) | |
java.base@11.0.6/java.io.BufferedOutputStream.write(BufferedOutputStream.java:95) | |
app//org.apache.lucene.store.OutputStreamIndexOutput.writeByte(OutputStreamIndexOutput.java:47) | |
app//org.elasticsearch.common.lucene.store.FilterIndexOutput.writeByte(FilterIndexOutput.java:54) | |
app//org.elasticsearch.index.store.ByteSizeCachingDirectory$2.writeByte(ByteSizeCachingDirectory.java:154) | |
app//org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:70) | |
app//org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:212) | |
app//org.apache.lucene.codecs.lucene80.IndexedDISI.flush(IndexedDISI.java:121) | |
app//org.apache.lucene.codecs.lucene80.IndexedDISI.writeBitSet(IndexedDISI.java:203) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:221) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118) | |
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109) | |
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373) | |
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587) | |
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230) | |
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311) | |
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262) | |
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112) | |
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225) | |
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625) | |
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604) | |
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304) | |
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840) | |
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111) | |
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972) | |
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144) | |
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
unique snapshot | |
app//org.apache.lucene.codecs.lucene80.IndexedDISI.writeBitSet(IndexedDISI.java:203) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:221) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118) | |
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109) | |
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373) | |
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587) | |
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230) | |
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311) | |
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262) | |
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112) | |
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225) | |
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625) | |
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604) | |
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304) | |
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840) | |
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111) | |
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972) | |
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144) | |
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
::: {741248536f8a624a46358affbf3940d9}{lunuJX_4QM-QSGxUuAK6aQ}{K6N5Dqy9TUOjP5v5nWfonw}{x.x.x.x}{x.x.x.x:9300}{dir}{distributed_snapshot_deletion_enabled=true, zone=us-east-1d, cross_cluster_transport_address=x.x.x.x} | |
Hot threads at 2021-01-15T01:04:30.188Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
100.2% (500.8ms out of 500ms) cpu usage by thread 'elasticsearch[741248536f8a624a46358affbf3940d9][[people][3]: Lucene Merge Thread #3804]' | |
2/10 snapshots sharing following 13 elements | |
app//org.apache.lucene.util.compress.LZ4.encodeSequence(LZ4.java:177) | |
app//org.apache.lucene.util.compress.LZ4.compress(LZ4.java:441) | |
app//org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:165) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:229) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:159) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:518) | |
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:229) | |
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106) | |
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463) | |
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625) | |
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662) | |
2/10 snapshots sharing following 13 elements | |
app//org.apache.lucene.util.compress.LZ4.decompress(LZ4.java:103) | |
app//org.apache.lucene.codecs.compressing.CompressionMode$4.decompress(CompressionMode.java:139) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader$BlockState.doReset(CompressingStoredFieldsReader.java:480) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader$BlockState.reset(CompressingStoredFieldsReader.java:393) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.document(CompressingStoredFieldsReader.java:572) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:514) | |
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:229) | |
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106) | |
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463) | |
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625) | |
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662) | |
2/10 snapshots sharing following 14 elements | |
app//org.apache.lucene.util.compress.LZ4.encodeLiterals(LZ4.java:151) | |
app//org.apache.lucene.util.compress.LZ4.encodeSequence(LZ4.java:172) | |
app//org.apache.lucene.util.compress.LZ4.compress(LZ4.java:441) | |
app//org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:165) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:229) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:159) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:518) | |
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:229) | |
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106) | |
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463) | |
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625) | |
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662) | |
3/10 snapshots sharing following 17 elements | |
app//org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
app//org.elasticsearch.common.lucene.store.FilterIndexOutput.writeBytes(FilterIndexOutput.java:59) | |
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:73) | |
app//org.apache.lucene.util.compress.LZ4.encodeLiterals(LZ4.java:159) | |
app//org.apache.lucene.util.compress.LZ4.encodeSequence(LZ4.java:172) | |
app//org.apache.lucene.util.compress.LZ4.compress(LZ4.java:441) | |
app//org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:165) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:229) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:159) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:518) | |
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:229) | |
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106) | |
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463) | |
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625) | |
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662) | |
unique snapshot | |
app//org.apache.lucene.util.compress.LZ4.decompress(LZ4.java:112) | |
app//org.apache.lucene.codecs.compressing.CompressionMode$4.decompress(CompressionMode.java:139) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader$BlockState.doReset(CompressingStoredFieldsReader.java:480) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader$BlockState.reset(CompressingStoredFieldsReader.java:393) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.document(CompressingStoredFieldsReader.java:572) | |
app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:514) | |
app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:229) | |
app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106) | |
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463) | |
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625) | |
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662) | |
64.1% (320.4ms out of 500ms) cpu usage by thread 'elasticsearch[741248536f8a624a46358affbf3940d9][refresh][T#1]' | |
2/10 snapshots sharing following 35 elements | |
app//org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:212) | |
app//org.apache.lucene.codecs.lucene80.IndexedDISI.flush(IndexedDISI.java:121) | |
app//org.apache.lucene.codecs.lucene80.IndexedDISI.writeBitSet(IndexedDISI.java:203) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:221) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118) | |
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109) | |
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373) | |
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587) | |
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230) | |
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311) | |
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262) | |
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112) | |
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225) | |
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625) | |
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604) | |
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304) | |
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840) | |
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111) | |
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972) | |
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144) | |
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
3/10 snapshots sharing following 39 elements | |
app//org.apache.lucene.codecs.lucene80.IndexedDISI.advance(IndexedDISI.java:384) | |
app//org.apache.lucene.codecs.lucene80.IndexedDISI.nextDoc(IndexedDISI.java:459) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$SparseNumericDocValues.nextDoc(Lucene80DocValuesProducer.java:444) | |
app//org.apache.lucene.index.ReadersAndUpdates$MergedDocValues.nextDoc(ReadersAndUpdates.java:469) | |
app//org.apache.lucene.index.ReadersAndUpdates$2$1.nextDoc(ReadersAndUpdates.java:396) | |
app//org.apache.lucene.index.SingletonSortedNumericDocValues.nextDoc(SingletonSortedNumericDocValues.java:53) | |
app//org.apache.lucene.codecs.lucene80.IndexedDISI.writeBitSet(IndexedDISI.java:196) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:221) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118) | |
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109) | |
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373) | |
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587) | |
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230) | |
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311) | |
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262) | |
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112) | |
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225) | |
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625) | |
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604) | |
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304) | |
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840) | |
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111) | |
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972) | |
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144) | |
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
2/10 snapshots sharing following 34 elements | |
app//org.apache.lucene.index.SingletonSortedNumericDocValues.nextDoc(SingletonSortedNumericDocValues.java:53) | |
app//org.apache.lucene.codecs.lucene80.IndexedDISI.writeBitSet(IndexedDISI.java:196) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:221) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118) | |
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109) | |
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373) | |
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587) | |
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230) | |
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311) | |
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262) | |
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112) | |
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225) | |
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625) | |
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604) | |
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304) | |
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840) | |
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111) | |
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972) | |
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144) | |
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
2/10 snapshots sharing following 33 elements | |
app//org.apache.lucene.codecs.lucene80.IndexedDISI.writeBitSet(IndexedDISI.java:203) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:221) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118) | |
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109) | |
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373) | |
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587) | |
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230) | |
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311) | |
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262) | |
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112) | |
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225) | |
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625) | |
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604) | |
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304) | |
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840) | |
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111) | |
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972) | |
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144) | |
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
unique snapshot | |
app//org.apache.lucene.codecs.lucene80.IndexedDISI.writeBitSet(IndexedDISI.java:196) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeValues(Lucene80DocValuesConsumer.java:221) | |
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addNumericField(Lucene80DocValuesConsumer.java:118) | |
app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:109) | |
app//org.apache.lucene.index.ReadersAndUpdates.handleDVUpdates(ReadersAndUpdates.java:373) | |
app//org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:587) | |
app//org.apache.lucene.index.ReaderPool.writeAllDocValuesUpdates(ReaderPool.java:230) | |
app//org.apache.lucene.index.IndexWriter.writeReaderPool(IndexWriter.java:3311) | |
app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:520) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272) | |
app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262) | |
app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112) | |
app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66) | |
app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362) | |
app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344) | |
app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225) | |
app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625) | |
app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604) | |
app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3304) | |
app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:840) | |
app//org.elasticsearch.index.IndexService.access$200(IndexService.java:111) | |
app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:972) | |
app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144) | |
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:645) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) | |
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) | |
java.base@11.0.6/java.lang.Thread.run(Thread.java:834) | |
0.7% (3.3ms out of 500ms) cpu usage by thread 'elasticsearch[741248536f8a624a46358affbf3940d9][[people][3]: Lucene Merge Thread #3794]' | |
10/10 snapshots sharing following 16 elements | |
java.base@11.0.6/jdk.internal.misc.Unsafe.park(Native Method) | |
java.base@11.0.6/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) | |
java.base@11.0.6/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) | |
app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:156) | |
app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148) | |
app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93) | |
app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78) | |
app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72) | |
app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278) | |
app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:96) | |
app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5014) | |
app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4527) | |
app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625) | |
app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101) | |
app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment