Skip to content

Instantly share code, notes, and snippets.

@HAKANMAZI

HAKANMAZI/gist:fee969b958e51591c1437be07ff13129 Secret

Created January 8, 2021 11:53
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save HAKANMAZI/fee969b958e51591c1437be07ff13129 to your computer and use it in GitHub Desktop.
Save HAKANMAZI/fee969b958e51591c1437be07ff13129 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
gistfile1.txt
::: {data-9}{2nEK_Bt1R5yfObiZTBha2Q}{i1xHvs2dS02GO1hkvDIFQg}{192.168.3.57}{192.168.3.57:9300}{dil}{ml.machine_memory=270435311616, ml.max_open_jobs=20, xpack.installed=true, disk_type=ssd}
Hot threads at 2021-01-08T08:24:49.618Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
::: {data-7}{U8_onApfRnG7fpToARusIw}{jvUfxK92RGmeWaNP7of13A}{192.168.3.63}{192.168.3.63:9300}{dil}{ml.machine_memory=270413611008, ml.max_open_jobs=20, xpack.installed=true, disk_type=ssd}
Hot threads at 2021-01-08T08:24:49.618Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
102.4% (511.9ms out of 500ms) cpu usage by thread 'elasticsearch[data-7][search][T#31]'
5/10 snapshots sharing following 27 elements
app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:253)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4365/0x0000000801868440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3707/0x0000000801660040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4375/0x000000080186ac40.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
2/10 snapshots sharing following 31 elements
app//org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:40)
app//org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:108)
app//org.apache.lucene.index.OrdinalMap$3.get(OrdinalMap.java:312)
app//org.apache.lucene.search.join.GlobalOrdinalsWithScoreQuery$OrdinalMapScorer$1.matches(GlobalOrdinalsWithScoreQuery.java:214)
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4365/0x0000000801868440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3707/0x0000000801660040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4375/0x000000080186ac40.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
3/10 snapshots sharing following 27 elements
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4365/0x0000000801868440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3707/0x0000000801660040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4375/0x000000080186ac40.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
102.4% (511.8ms out of 500ms) cpu usage by thread 'elasticsearch[data-7][search][T#62]'
3/10 snapshots sharing following 27 elements
app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:253)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4365/0x0000000801868440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3707/0x0000000801660040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4375/0x000000080186ac40.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
5/10 snapshots sharing following 27 elements
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4365/0x0000000801868440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3707/0x0000000801660040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4375/0x000000080186ac40.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
2/10 snapshots sharing following 10 elements
java.base@13.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@13.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
java.base@13.0.1/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:743)
java.base@13.0.1/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684)
java.base@13.0.1/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1366)
app//org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
102.4% (511.8ms out of 500ms) cpu usage by thread 'elasticsearch[data-7][search][T#58]'
2/10 snapshots sharing following 27 elements
app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:253)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4365/0x0000000801868440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3707/0x0000000801660040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4375/0x000000080186ac40.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
2/10 snapshots sharing following 31 elements
app//org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:40)
app//org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:108)
app//org.apache.lucene.index.OrdinalMap$3.get(OrdinalMap.java:312)
app//org.apache.lucene.search.join.GlobalOrdinalsWithScoreQuery$OrdinalMapScorer$1.matches(GlobalOrdinalsWithScoreQuery.java:214)
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4365/0x0000000801868440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3707/0x0000000801660040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4375/0x000000080186ac40.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
2/10 snapshots sharing following 27 elements
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4365/0x0000000801868440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3707/0x0000000801660040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4375/0x000000080186ac40.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
4/10 snapshots sharing following 10 elements
java.base@13.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@13.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
java.base@13.0.1/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:743)
java.base@13.0.1/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684)
java.base@13.0.1/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1366)
app//org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
::: {data-2}{vKA6qG2WRr2VJnK4wNVYOA}{V6tcNEsnSjm7VHVxDiIiHw}{192.168.3.69}{192.168.3.69:9300}{dil}{ml.machine_memory=405356605440, ml.max_open_jobs=20, xpack.installed=true, disk_type=ssd}
Hot threads at 2021-01-08T08:24:49.618Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
101.4% (506.9ms out of 500ms) cpu usage by thread 'elasticsearch[data-2][search][T#27]'
6/10 snapshots sharing following 27 elements
app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:253)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4747/0x000000080191bc40.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3698/0x0000000801657c40.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4795/0x0000000801924040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
2/10 snapshots sharing following 30 elements
app//org.apache.lucene.codecs.lucene80.IndexedDISI.advanceExact(IndexedDISI.java:399)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$20.advanceExact(Lucene80DocValuesProducer.java:839)
app//org.apache.lucene.search.join.GlobalOrdinalsWithScoreQuery$OrdinalMapScorer$1.matches(GlobalOrdinalsWithScoreQuery.java:212)
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4747/0x000000080191bc40.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3698/0x0000000801657c40.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4795/0x0000000801924040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
2/10 snapshots sharing following 27 elements
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4747/0x000000080191bc40.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3698/0x0000000801657c40.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4795/0x0000000801924040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
101.4% (506.9ms out of 500ms) cpu usage by thread 'elasticsearch[data-2][search][T#39]'
2/10 snapshots sharing following 28 elements
app//org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:200)
app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:253)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4747/0x000000080191bc40.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3698/0x0000000801657c40.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4795/0x0000000801924040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
3/10 snapshots sharing following 30 elements
app//org.apache.lucene.codecs.lucene80.IndexedDISI.advanceExact(IndexedDISI.java:399)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$20.advanceExact(Lucene80DocValuesProducer.java:839)
app//org.apache.lucene.search.join.GlobalOrdinalsWithScoreQuery$OrdinalMapScorer$1.matches(GlobalOrdinalsWithScoreQuery.java:212)
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4747/0x000000080191bc40.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3698/0x0000000801657c40.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4795/0x0000000801924040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
2/10 snapshots sharing following 31 elements
app//org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:40)
app//org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:108)
app//org.apache.lucene.index.OrdinalMap$3.get(OrdinalMap.java:312)
app//org.apache.lucene.search.join.GlobalOrdinalsWithScoreQuery$OrdinalMapScorer$1.matches(GlobalOrdinalsWithScoreQuery.java:214)
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4747/0x000000080191bc40.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3698/0x0000000801657c40.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4795/0x0000000801924040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
2/10 snapshots sharing following 29 elements
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$20.ordValue(Lucene80DocValuesProducer.java:844)
app//org.apache.lucene.search.join.GlobalOrdinalsWithScoreQuery$OrdinalMapScorer$1.matches(GlobalOrdinalsWithScoreQuery.java:213)
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4747/0x000000080191bc40.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3698/0x0000000801657c40.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4795/0x0000000801924040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
unique snapshot
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4747/0x000000080191bc40.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3698/0x0000000801657c40.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4795/0x0000000801924040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
101.4% (506.9ms out of 500ms) cpu usage by thread 'elasticsearch[data-2][search][T#68]'
10/10 snapshots sharing following 25 elements
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4747/0x000000080191bc40.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3698/0x0000000801657c40.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4795/0x0000000801924040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
::: {data-4}{NVXXnkg9RcS1LJHbxE4JWw}{I7q-NniJRnS8HDUO43x_QQ}{192.168.3.66}{192.168.3.66:9300}{dil}{ml.machine_memory=405356519424, ml.max_open_jobs=20, xpack.installed=true, disk_type=ssd}
Hot threads at 2021-01-08T08:24:49.617Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
101.6% (507.8ms out of 500ms) cpu usage by thread 'elasticsearch[data-4][search][T#27]'
3/10 snapshots sharing following 30 elements
app//org.apache.lucene.codecs.lucene80.IndexedDISI.advanceExact(IndexedDISI.java:399)
app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$20.advanceExact(Lucene80DocValuesProducer.java:839)
app//org.apache.lucene.search.join.GlobalOrdinalsWithScoreQuery$OrdinalMapScorer$1.matches(GlobalOrdinalsWithScoreQuery.java:212)
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4536/0x00000008016a0440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3671/0x000000080164e440.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4546/0x0000000801654040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
3/10 snapshots sharing following 31 elements
app//org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:40)
app//org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:108)
app//org.apache.lucene.index.OrdinalMap$3.get(OrdinalMap.java:312)
app//org.apache.lucene.search.join.GlobalOrdinalsWithScoreQuery$OrdinalMapScorer$1.matches(GlobalOrdinalsWithScoreQuery.java:214)
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4536/0x00000008016a0440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3671/0x000000080164e440.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4546/0x0000000801654040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
3/10 snapshots sharing following 27 elements
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4536/0x00000008016a0440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3671/0x000000080164e440.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4546/0x0000000801654040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
unique snapshot
app//org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:213)
app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:253)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4536/0x00000008016a0440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3671/0x000000080164e440.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4546/0x0000000801654040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
101.6% (507.8ms out of 500ms) cpu usage by thread 'elasticsearch[data-4][search][T#47]'
2/10 snapshots sharing following 31 elements
app//org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:40)
app//org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:108)
app//org.apache.lucene.index.OrdinalMap$3.get(OrdinalMap.java:312)
app//org.apache.lucene.search.join.GlobalOrdinalsWithScoreQuery$OrdinalMapScorer$1.matches(GlobalOrdinalsWithScoreQuery.java:214)
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4536/0x00000008016a0440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3671/0x000000080164e440.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4546/0x0000000801654040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
2/10 snapshots sharing following 25 elements
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4536/0x00000008016a0440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3671/0x000000080164e440.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4546/0x0000000801654040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
6/10 snapshots sharing following 10 elements
java.base@13.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@13.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
java.base@13.0.1/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:743)
java.base@13.0.1/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684)
java.base@13.0.1/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1366)
app//org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
101.6% (507.8ms out of 500ms) cpu usage by thread 'elasticsearch[data-4][search][T#38]'
3/10 snapshots sharing following 27 elements
app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:253)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4536/0x00000008016a0440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3671/0x000000080164e440.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4546/0x0000000801654040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
2/10 snapshots sharing following 27 elements
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4536/0x00000008016a0440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3671/0x000000080164e440.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4546/0x0000000801654040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
5/10 snapshots sharing following 10 elements
java.base@13.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@13.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
java.base@13.0.1/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:743)
java.base@13.0.1/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684)
java.base@13.0.1/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1366)
app//org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
::: {master-2}{09WYCZWjSbWXZ7eqSxcShQ}{1cxh8TRSQ5y4lZhmAPWJGA}{192.168.3.60}{192.168.3.60:9300}{m}{xpack.installed=true}
Hot threads at 2021-01-08T08:24:49.618Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
::: {data-6}{RViUJd6gQJui6Hu5clZh1g}{r5mcTYo_RAiSvwTNpVb1SQ}{192.168.3.64}{192.168.3.64:9300}{dil}{ml.machine_memory=270435397632, ml.max_open_jobs=20, xpack.installed=true, disk_type=ssd}
Hot threads at 2021-01-08T08:24:49.618Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
::: {data-3}{pro3gLa8TuqWDxjBjVTZVw}{F5OQjvP8Q9GOgFMIK_fNpg}{192.168.3.67}{192.168.3.67:9300}{dil}{ml.machine_memory=270073495552, ml.max_open_jobs=20, xpack.installed=true, disk_type=ssd}
Hot threads at 2021-01-08T08:24:49.618Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
102.2% (510.8ms out of 500ms) cpu usage by thread 'elasticsearch[data-3][search][T#52]'
4/10 snapshots sharing following 27 elements
app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:253)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4398/0x00000008018a6440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3669/0x000000080164e040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4414/0x00000008018a2440.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
6/10 snapshots sharing following 27 elements
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4398/0x00000008018a6440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3669/0x000000080164e040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4414/0x00000008018a2440.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
102.1% (510.6ms out of 500ms) cpu usage by thread 'elasticsearch[data-3][search][T#71]'
6/10 snapshots sharing following 27 elements
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4398/0x00000008018a6440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3669/0x000000080164e040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4414/0x00000008018a2440.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
4/10 snapshots sharing following 27 elements
app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:253)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4398/0x00000008018a6440.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3669/0x000000080164e040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4414/0x00000008018a2440.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
::: {master-3}{wHJJAON-TS-9uMYtbisMig}{WYIoYMs3QhS3sVGCi0_Ukw}{192.168.3.59}{192.168.3.59:9300}{m}{xpack.installed=true, disk_type=ssd}
Hot threads at 2021-01-08T08:24:49.618Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
::: {data-11}{nNuEnLSOQMWbx7C6u37bfQ}{Pm9fC9rdQrGoBFenTnSVRg}{192.168.3.71}{192.168.3.71:9300}{dil}{ml.machine_memory=67508068352, ml.max_open_jobs=20, xpack.installed=true, disk_type=ssd}
Hot threads at 2021-01-08T08:24:49.619Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
103.0% (514.9ms out of 500ms) cpu usage by thread 'elasticsearch[data-11][search][T#18]'
5/10 snapshots sharing following 31 elements
app//org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:40)
app//org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:108)
app//org.apache.lucene.index.OrdinalMap$3.get(OrdinalMap.java:312)
app//org.apache.lucene.search.join.GlobalOrdinalsWithScoreQuery$OrdinalMapScorer$1.matches(GlobalOrdinalsWithScoreQuery.java:214)
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4448/0x0000000801880c40.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3673/0x000000080164f040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4453/0x0000000801882040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
3/10 snapshots sharing following 28 elements
app//org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:213)
app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:253)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4448/0x0000000801880c40.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3673/0x000000080164f040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4453/0x0000000801882040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
2/10 snapshots sharing following 25 elements
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4448/0x0000000801880c40.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3673/0x000000080164f040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4453/0x0000000801882040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
103.0% (514.9ms out of 500ms) cpu usage by thread 'elasticsearch[data-11][search][T#51]'
3/10 snapshots sharing following 31 elements
app//org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:40)
app//org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:108)
app//org.apache.lucene.index.OrdinalMap$3.get(OrdinalMap.java:312)
app//org.apache.lucene.search.join.GlobalOrdinalsWithScoreQuery$OrdinalMapScorer$1.matches(GlobalOrdinalsWithScoreQuery.java:214)
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4448/0x0000000801880c40.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3673/0x000000080164f040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4453/0x0000000801882040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
7/10 snapshots sharing following 26 elements
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:253)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4448/0x0000000801880c40.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3673/0x000000080164f040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4453/0x0000000801882040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
103.0% (514.8ms out of 500ms) cpu usage by thread 'elasticsearch[data-11][search][T#91]'
4/10 snapshots sharing following 27 elements
app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:253)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4448/0x0000000801880c40.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3673/0x000000080164f040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4453/0x0000000801882040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
3/10 snapshots sharing following 31 elements
app//org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:40)
app//org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:108)
app//org.apache.lucene.index.OrdinalMap$3.get(OrdinalMap.java:312)
app//org.apache.lucene.search.join.GlobalOrdinalsWithScoreQuery$OrdinalMapScorer$1.matches(GlobalOrdinalsWithScoreQuery.java:214)
app//org.apache.lucene.search.ConjunctionDISI$ConjunctionTwoPhaseIterator.matches(ConjunctionDISI.java:345)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:250)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4448/0x0000000801880c40.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3673/0x000000080164f040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4453/0x0000000801882040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
3/10 snapshots sharing following 25 elements
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.elasticsearch.search.internal.ContextIndexSearcher.searchInternal(ContextIndexSearcher.java:193)
app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:174)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:270)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:335)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:355)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4448/0x0000000801880c40.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3673/0x000000080164f040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4453/0x0000000801882040.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
::: {master-1}{KQhsToi9TBynzxYoJS-iGw}{p87epSOWSHOrEYaFCIrQbA}{192.168.3.62}{192.168.3.62:9300}{m}{xpack.installed=true}
Hot threads at 2021-01-08T08:24:49.619Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
::: {data-1}{_BUmAFaYTGi-6aaYnPlqvw}{R8bdK4JlRVK7CpqNOk4Qeg}{192.168.3.68}{192.168.3.68:9300}{dil}{ml.machine_memory=270073581568, ml.max_open_jobs=20, xpack.installed=true, disk_type=ssd}
Hot threads at 2021-01-08T08:24:49.619Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
::: {data-10}{U_Gj7l6MToOyiHIVf8PKeQ}{XoMK6fm8S9i28CT69aKt4Q}{192.168.3.72}{192.168.3.72:9300}{dil}{ml.machine_memory=67515105280, ml.max_open_jobs=20, xpack.installed=true, disk_type=ssd}
Hot threads at 2021-01-08T08:24:49.619Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
99.8% (498.8ms out of 500ms) cpu usage by thread 'elasticsearch[data-10][search][T#3]'
2/10 snapshots sharing following 34 elements
app//org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:40)
app//org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:108)
app//org.apache.lucene.index.OrdinalMap$3.get(OrdinalMap.java:312)
app//org.apache.lucene.search.join.GlobalOrdinalsWithScoreCollector$NoScore$1.collect(GlobalOrdinalsWithScoreCollector.java:256)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:267)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:218)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:651)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.apache.lucene.search.join.JoinUtil.createJoinQuery(JoinUtil.java:511)
org.elasticsearch.join.query.HasChildQueryBuilder$LateParsingQuery.rewrite(HasChildQueryBuilder.java:379)
app//org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:270)
app//org.apache.lucene.search.IndexSearcher.rewrite(IndexSearcher.java:666)
app//org.elasticsearch.search.internal.ContextIndexSearcher.rewrite(ContextIndexSearcher.java:105)
app//org.elasticsearch.search.DefaultSearchContext.preProcess(DefaultSearchContext.java:262)
app//org.elasticsearch.search.query.QueryPhase.preProcess(QueryPhase.java:91)
app//org.elasticsearch.search.SearchService.createContext(SearchService.java:598)
app//org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:545)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:348)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4550/0x000000080188a840.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3709/0x0000000801660040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4555/0x000000080188bc40.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
8/10 snapshots sharing following 30 elements
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:267)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:218)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:651)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.apache.lucene.search.join.JoinUtil.createJoinQuery(JoinUtil.java:511)
org.elasticsearch.join.query.HasChildQueryBuilder$LateParsingQuery.rewrite(HasChildQueryBuilder.java:379)
app//org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:270)
app//org.apache.lucene.search.IndexSearcher.rewrite(IndexSearcher.java:666)
app//org.elasticsearch.search.internal.ContextIndexSearcher.rewrite(ContextIndexSearcher.java:105)
app//org.elasticsearch.search.DefaultSearchContext.preProcess(DefaultSearchContext.java:262)
app//org.elasticsearch.search.query.QueryPhase.preProcess(QueryPhase.java:91)
app//org.elasticsearch.search.SearchService.createContext(SearchService.java:598)
app//org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:545)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:348)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4550/0x000000080188a840.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3709/0x0000000801660040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4555/0x000000080188bc40.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
94.1% (470.3ms out of 500ms) cpu usage by thread 'elasticsearch[data-10][search][T#34]'
2/10 snapshots sharing following 31 elements
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:265)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:218)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:651)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.apache.lucene.search.join.JoinUtil.createJoinQuery(JoinUtil.java:511)
org.elasticsearch.join.query.HasChildQueryBuilder$LateParsingQuery.rewrite(HasChildQueryBuilder.java:379)
app//org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:270)
app//org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:270)
app//org.apache.lucene.search.IndexSearcher.rewrite(IndexSearcher.java:666)
app//org.elasticsearch.search.internal.ContextIndexSearcher.rewrite(ContextIndexSearcher.java:105)
app//org.elasticsearch.search.DefaultSearchContext.preProcess(DefaultSearchContext.java:262)
app//org.elasticsearch.search.query.QueryPhase.preProcess(QueryPhase.java:91)
app//org.elasticsearch.search.SearchService.createContext(SearchService.java:598)
app//org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:545)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:348)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4550/0x000000080188a840.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3709/0x0000000801660040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4555/0x000000080188bc40.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
5/10 snapshots sharing following 32 elements
app//org.apache.lucene.search.join.GlobalOrdinalsWithScoreCollector$NoScore$1.collect(GlobalOrdinalsWithScoreCollector.java:256)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:267)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:218)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:651)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.apache.lucene.search.join.JoinUtil.createJoinQuery(JoinUtil.java:511)
org.elasticsearch.join.query.HasChildQueryBuilder$LateParsingQuery.rewrite(HasChildQueryBuilder.java:379)
app//org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:270)
app//org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:270)
app//org.apache.lucene.search.IndexSearcher.rewrite(IndexSearcher.java:666)
app//org.elasticsearch.search.internal.ContextIndexSearcher.rewrite(ContextIndexSearcher.java:105)
app//org.elasticsearch.search.DefaultSearchContext.preProcess(DefaultSearchContext.java:262)
app//org.elasticsearch.search.query.QueryPhase.preProcess(QueryPhase.java:91)
app//org.elasticsearch.search.SearchService.createContext(SearchService.java:598)
app//org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:545)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:348)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4550/0x000000080188a840.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3709/0x0000000801660040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4555/0x000000080188bc40.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
3/10 snapshots sharing following 31 elements
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:267)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:218)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:651)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.apache.lucene.search.join.JoinUtil.createJoinQuery(JoinUtil.java:511)
org.elasticsearch.join.query.HasChildQueryBuilder$LateParsingQuery.rewrite(HasChildQueryBuilder.java:379)
app//org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:270)
app//org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:270)
app//org.apache.lucene.search.IndexSearcher.rewrite(IndexSearcher.java:666)
app//org.elasticsearch.search.internal.ContextIndexSearcher.rewrite(ContextIndexSearcher.java:105)
app//org.elasticsearch.search.DefaultSearchContext.preProcess(DefaultSearchContext.java:262)
app//org.elasticsearch.search.query.QueryPhase.preProcess(QueryPhase.java:91)
app//org.elasticsearch.search.SearchService.createContext(SearchService.java:598)
app//org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:545)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:348)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4550/0x000000080188a840.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3709/0x0000000801660040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4555/0x000000080188bc40.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
37.5% (187.7ms out of 500ms) cpu usage by thread 'elasticsearch[data-10][search][T#60]'
8/10 snapshots sharing following 31 elements
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:267)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:218)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:651)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.apache.lucene.search.join.JoinUtil.createJoinQuery(JoinUtil.java:511)
org.elasticsearch.join.query.HasChildQueryBuilder$LateParsingQuery.rewrite(HasChildQueryBuilder.java:379)
app//org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:270)
app//org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:270)
app//org.apache.lucene.search.IndexSearcher.rewrite(IndexSearcher.java:666)
app//org.elasticsearch.search.internal.ContextIndexSearcher.rewrite(ContextIndexSearcher.java:105)
app//org.elasticsearch.search.DefaultSearchContext.preProcess(DefaultSearchContext.java:262)
app//org.elasticsearch.search.query.QueryPhase.preProcess(QueryPhase.java:91)
app//org.elasticsearch.search.SearchService.createContext(SearchService.java:598)
app//org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:545)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:348)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4550/0x000000080188a840.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3709/0x0000000801660040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4555/0x000000080188bc40.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
2/10 snapshots sharing following 31 elements
app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:265)
app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:218)
app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:651)
app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:440)
app//org.apache.lucene.search.join.JoinUtil.createJoinQuery(JoinUtil.java:511)
org.elasticsearch.join.query.HasChildQueryBuilder$LateParsingQuery.rewrite(HasChildQueryBuilder.java:379)
app//org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:270)
app//org.apache.lucene.search.BooleanQuery.rewrite(BooleanQuery.java:270)
app//org.apache.lucene.search.IndexSearcher.rewrite(IndexSearcher.java:666)
app//org.elasticsearch.search.internal.ContextIndexSearcher.rewrite(ContextIndexSearcher.java:105)
app//org.elasticsearch.search.DefaultSearchContext.preProcess(DefaultSearchContext.java:262)
app//org.elasticsearch.search.query.QueryPhase.preProcess(QueryPhase.java:91)
app//org.elasticsearch.search.SearchService.createContext(SearchService.java:598)
app//org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:545)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:348)
app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$1(SearchService.java:340)
app//org.elasticsearch.search.SearchService$$Lambda$4550/0x000000080188a840.apply(Unknown Source)
app//org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:145)
app//org.elasticsearch.action.ActionListener$$Lambda$3709/0x0000000801660040.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
app//org.elasticsearch.search.SearchService.lambda$rewriteShardRequest$7(SearchService.java:1043)
app//org.elasticsearch.search.SearchService$$Lambda$4555/0x000000080188bc40.accept(Unknown Source)
app//org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:45)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@13.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@13.0.1/java.lang.Thread.run(Thread.java:830)
::: {data-5}{jFQih3_gQ8-MdAzbTwlHFg}{dHpfDQZWRMmH-_SoBfbrfw}{192.168.3.65}{192.168.3.65:9300}{dil}{ml.machine_memory=270435397632, ml.max_open_jobs=20, xpack.installed=true, disk_type=ssd}
Hot threads at 2021-01-08T08:24:49.619Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment