Skip to content

Instantly share code, notes, and snippets.

@smlbiobot

smlbiobot/esnode4-hotthreads.txt Secret

Created June 20, 2021 13:37
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save smlbiobot/4300885e86a0b2af4f8898da0fed9ac0 to your computer and use it in GitHub Desktop.
Save smlbiobot/4300885e86a0b2af4f8898da0fed9ac0 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
esnode4-hotthreads.txt
::: {esnode4}{LSKHTPRoTCOWAKJAYQB70Q}{q_eVAy06Roy5SkV4UlaFeA}{xxx.xxx.xxx.xxx}{xxx.xxx.xxx.xxx:9300}{ml.machine_memory=33728798720, ml.max_open_jobs=20, xpack.installed=true, box_type=hot, ml.enabled=true}
Hot threads at 2021-06-20T13:34:49.390Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
83.4% (416.7ms out of 500ms) cpu usage by thread 'elasticsearch[esnode4][search][T#16]'
4/10 snapshots sharing following 46 elements
app//org.apache.lucene.util.PriorityQueue.downHeap(PriorityQueue.java:275)
app//org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:202)
app//org.apache.lucene.index.OrdinalMap.<init>(OrdinalMap.java:261)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:147)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder.build(GlobalOrdinalsBuilder.java:65)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:130)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:47)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:172)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$3582/1911147663.load(Unknown Source)
app//org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:169)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:118)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals$FieldData.globalOrdinalsValues(ValuesSource.java:151)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals.globalMaxOrd(ValuesSource.java:124)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.getMaxOrd(TermsAggregatorFactory.java:214)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory$ExecutionMode$2.create(TermsAggregatorFactory.java:259)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.doCreateInternal(TermsAggregatorFactory.java:154)
app//org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory.createInternal(ValuesSourceAggregatorFactory.java:59)
app//org.elasticsearch.search.aggregations.AggregatorFactory.create(AggregatorFactory.java:216)
app//org.elasticsearch.search.aggregations.AggregatorFactories.createTopLevelAggregators(AggregatorFactories.java:217)
app//org.elasticsearch.search.aggregations.AggregationPhase.preProcess(AggregationPhase.java:55)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:112)
app//org.elasticsearch.indices.IndicesService.lambda$loadIntoContext$17(IndicesService.java:1252)
app//org.elasticsearch.indices.IndicesService$$Lambda$3475/544101620.accept(Unknown Source)
app//org.elasticsearch.indices.IndicesService.lambda$cacheShardLevelResult$18(IndicesService.java:1308)
app//org.elasticsearch.indices.IndicesService$$Lambda$3476/740220962.get(Unknown Source)
app//org.elasticsearch.indices.IndicesRequestCache$Loader.load(IndicesRequestCache.java:164)
app//org.elasticsearch.indices.IndicesRequestCache$Loader.load(IndicesRequestCache.java:147)
app//org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
app//org.elasticsearch.indices.IndicesRequestCache.getOrCompute(IndicesRequestCache.java:119)
app//org.elasticsearch.indices.IndicesService.cacheShardLevelResult(IndicesService.java:1314)
app//org.elasticsearch.indices.IndicesService.loadIntoContext(IndicesService.java:1250)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:348)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
app//org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
app//org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1117)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:778)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1135)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@10.0.1/java.lang.Thread.run(Thread.java:844)
4/10 snapshots sharing following 45 elements
app//org.apache.lucene.index.OrdinalMap$TermsEnumIndex.next(OrdinalMap.java:59)
app//org.apache.lucene.index.OrdinalMap.<init>(OrdinalMap.java:255)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:147)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder.build(GlobalOrdinalsBuilder.java:65)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:130)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:47)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:172)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$3582/1911147663.load(Unknown Source)
app//org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:169)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:118)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals$FieldData.globalOrdinalsValues(ValuesSource.java:151)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals.globalMaxOrd(ValuesSource.java:124)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.getMaxOrd(TermsAggregatorFactory.java:214)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory$ExecutionMode$2.create(TermsAggregatorFactory.java:259)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.doCreateInternal(TermsAggregatorFactory.java:154)
app//org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory.createInternal(ValuesSourceAggregatorFactory.java:59)
app//org.elasticsearch.search.aggregations.AggregatorFactory.create(AggregatorFactory.java:216)
app//org.elasticsearch.search.aggregations.AggregatorFactories.createTopLevelAggregators(AggregatorFactories.java:217)
app//org.elasticsearch.search.aggregations.AggregationPhase.preProcess(AggregationPhase.java:55)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:112)
app//org.elasticsearch.indices.IndicesService.lambda$loadIntoContext$17(IndicesService.java:1252)
app//org.elasticsearch.indices.IndicesService$$Lambda$3475/544101620.accept(Unknown Source)
app//org.elasticsearch.indices.IndicesService.lambda$cacheShardLevelResult$18(IndicesService.java:1308)
app//org.elasticsearch.indices.IndicesService$$Lambda$3476/740220962.get(Unknown Source)
app//org.elasticsearch.indices.IndicesRequestCache$Loader.load(IndicesRequestCache.java:164)
app//org.elasticsearch.indices.IndicesRequestCache$Loader.load(IndicesRequestCache.java:147)
app//org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
app//org.elasticsearch.indices.IndicesRequestCache.getOrCompute(IndicesRequestCache.java:119)
app//org.elasticsearch.indices.IndicesService.cacheShardLevelResult(IndicesService.java:1314)
app//org.elasticsearch.indices.IndicesService.loadIntoContext(IndicesService.java:1250)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:348)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
app//org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
app//org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1117)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:778)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1135)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@10.0.1/java.lang.Thread.run(Thread.java:844)
2/10 snapshots sharing following 43 elements
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:147)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder.build(GlobalOrdinalsBuilder.java:65)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:130)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:47)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:172)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$3582/1911147663.load(Unknown Source)
app//org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:169)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:118)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals$FieldData.globalOrdinalsValues(ValuesSource.java:151)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals.globalMaxOrd(ValuesSource.java:124)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.getMaxOrd(TermsAggregatorFactory.java:214)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory$ExecutionMode$2.create(TermsAggregatorFactory.java:259)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.doCreateInternal(TermsAggregatorFactory.java:154)
app//org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory.createInternal(ValuesSourceAggregatorFactory.java:59)
app//org.elasticsearch.search.aggregations.AggregatorFactory.create(AggregatorFactory.java:216)
app//org.elasticsearch.search.aggregations.AggregatorFactories.createTopLevelAggregators(AggregatorFactories.java:217)
app//org.elasticsearch.search.aggregations.AggregationPhase.preProcess(AggregationPhase.java:55)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:112)
app//org.elasticsearch.indices.IndicesService.lambda$loadIntoContext$17(IndicesService.java:1252)
app//org.elasticsearch.indices.IndicesService$$Lambda$3475/544101620.accept(Unknown Source)
app//org.elasticsearch.indices.IndicesService.lambda$cacheShardLevelResult$18(IndicesService.java:1308)
app//org.elasticsearch.indices.IndicesService$$Lambda$3476/740220962.get(Unknown Source)
app//org.elasticsearch.indices.IndicesRequestCache$Loader.load(IndicesRequestCache.java:164)
app//org.elasticsearch.indices.IndicesRequestCache$Loader.load(IndicesRequestCache.java:147)
app//org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
app//org.elasticsearch.indices.IndicesRequestCache.getOrCompute(IndicesRequestCache.java:119)
app//org.elasticsearch.indices.IndicesService.cacheShardLevelResult(IndicesService.java:1314)
app//org.elasticsearch.indices.IndicesService.loadIntoContext(IndicesService.java:1250)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:348)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
app//org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
app//org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1117)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:778)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1135)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@10.0.1/java.lang.Thread.run(Thread.java:844)
83.3% (416.5ms out of 500ms) cpu usage by thread 'elasticsearch[esnode4][search][T#5]'
5/10 snapshots sharing following 35 elements
app//org.apache.lucene.index.OrdinalMap$TermsEnumIndex.next(OrdinalMap.java:59)
app//org.apache.lucene.index.OrdinalMap.<init>(OrdinalMap.java:255)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:147)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder.build(GlobalOrdinalsBuilder.java:65)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:130)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:47)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:172)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$3582/1911147663.load(Unknown Source)
app//org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:169)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:118)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals$FieldData.globalOrdinalsValues(ValuesSource.java:151)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals.globalMaxOrd(ValuesSource.java:124)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.getMaxOrd(TermsAggregatorFactory.java:214)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory$ExecutionMode$2.create(TermsAggregatorFactory.java:259)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.doCreateInternal(TermsAggregatorFactory.java:154)
app//org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory.createInternal(ValuesSourceAggregatorFactory.java:59)
app//org.elasticsearch.search.aggregations.AggregatorFactory.create(AggregatorFactory.java:216)
app//org.elasticsearch.search.aggregations.AggregatorFactories.createTopLevelAggregators(AggregatorFactories.java:217)
app//org.elasticsearch.search.aggregations.AggregationPhase.preProcess(AggregationPhase.java:55)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:112)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
app//org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
app//org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1117)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:778)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1135)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@10.0.1/java.lang.Thread.run(Thread.java:844)
3/10 snapshots sharing following 34 elements
app//org.apache.lucene.index.OrdinalMap.<init>(OrdinalMap.java:272)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:147)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder.build(GlobalOrdinalsBuilder.java:65)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:130)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:47)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:172)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$3582/1911147663.load(Unknown Source)
app//org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:169)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:118)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals$FieldData.globalOrdinalsValues(ValuesSource.java:151)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals.globalMaxOrd(ValuesSource.java:124)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.getMaxOrd(TermsAggregatorFactory.java:214)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory$ExecutionMode$2.create(TermsAggregatorFactory.java:259)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.doCreateInternal(TermsAggregatorFactory.java:154)
app//org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory.createInternal(ValuesSourceAggregatorFactory.java:59)
app//org.elasticsearch.search.aggregations.AggregatorFactory.create(AggregatorFactory.java:216)
app//org.elasticsearch.search.aggregations.AggregatorFactories.createTopLevelAggregators(AggregatorFactories.java:217)
app//org.elasticsearch.search.aggregations.AggregationPhase.preProcess(AggregationPhase.java:55)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:112)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
app//org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
app//org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1117)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:778)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1135)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@10.0.1/java.lang.Thread.run(Thread.java:844)
2/10 snapshots sharing following 33 elements
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:147)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder.build(GlobalOrdinalsBuilder.java:65)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:130)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:47)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:172)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$3582/1911147663.load(Unknown Source)
app//org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:169)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:118)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals$FieldData.globalOrdinalsValues(ValuesSource.java:151)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals.globalMaxOrd(ValuesSource.java:124)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.getMaxOrd(TermsAggregatorFactory.java:214)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory$ExecutionMode$2.create(TermsAggregatorFactory.java:259)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.doCreateInternal(TermsAggregatorFactory.java:154)
app//org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory.createInternal(ValuesSourceAggregatorFactory.java:59)
app//org.elasticsearch.search.aggregations.AggregatorFactory.create(AggregatorFactory.java:216)
app//org.elasticsearch.search.aggregations.AggregatorFactories.createTopLevelAggregators(AggregatorFactories.java:217)
app//org.elasticsearch.search.aggregations.AggregationPhase.preProcess(AggregationPhase.java:55)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:112)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
app//org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
app//org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1117)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:778)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1135)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@10.0.1/java.lang.Thread.run(Thread.java:844)
83.3% (416.4ms out of 500ms) cpu usage by thread 'elasticsearch[esnode4][search][T#2]'
3/10 snapshots sharing following 34 elements
app//org.apache.lucene.index.OrdinalMap.<init>(OrdinalMap.java:272)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:147)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder.build(GlobalOrdinalsBuilder.java:65)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:130)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:47)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:172)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$3582/1911147663.load(Unknown Source)
app//org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:169)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:118)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals$FieldData.globalOrdinalsValues(ValuesSource.java:151)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals.globalMaxOrd(ValuesSource.java:124)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.getMaxOrd(TermsAggregatorFactory.java:214)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory$ExecutionMode$2.create(TermsAggregatorFactory.java:259)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.doCreateInternal(TermsAggregatorFactory.java:154)
app//org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory.createInternal(ValuesSourceAggregatorFactory.java:59)
app//org.elasticsearch.search.aggregations.AggregatorFactory.create(AggregatorFactory.java:216)
app//org.elasticsearch.search.aggregations.AggregatorFactories.createTopLevelAggregators(AggregatorFactories.java:217)
app//org.elasticsearch.search.aggregations.AggregationPhase.preProcess(AggregationPhase.java:55)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:112)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
app//org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
app//org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1117)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:778)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1135)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@10.0.1/java.lang.Thread.run(Thread.java:844)
3/10 snapshots sharing following 36 elements
app//org.apache.lucene.util.PriorityQueue.downHeap(PriorityQueue.java:275)
app//org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:202)
app//org.apache.lucene.index.OrdinalMap.<init>(OrdinalMap.java:261)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:147)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder.build(GlobalOrdinalsBuilder.java:65)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:130)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:47)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:172)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$3582/1911147663.load(Unknown Source)
app//org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:169)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:118)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals$FieldData.globalOrdinalsValues(ValuesSource.java:151)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals.globalMaxOrd(ValuesSource.java:124)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.getMaxOrd(TermsAggregatorFactory.java:214)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory$ExecutionMode$2.create(TermsAggregatorFactory.java:259)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.doCreateInternal(TermsAggregatorFactory.java:154)
app//org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory.createInternal(ValuesSourceAggregatorFactory.java:59)
app//org.elasticsearch.search.aggregations.AggregatorFactory.create(AggregatorFactory.java:216)
app//org.elasticsearch.search.aggregations.AggregatorFactories.createTopLevelAggregators(AggregatorFactories.java:217)
app//org.elasticsearch.search.aggregations.AggregationPhase.preProcess(AggregationPhase.java:55)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:112)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
app//org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
app//org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1117)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:778)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1135)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@10.0.1/java.lang.Thread.run(Thread.java:844)
2/10 snapshots sharing following 39 elements
app//org.apache.lucene.store.ByteBufferGuard.getByte(ByteBufferGuard.java:98)
app//org.apache.lucene.store.ByteBufferIndexInput.readByte(ByteBufferIndexInput.java:70)
app//org.apache.lucene.store.DataInput.readVInt(DataInput.java:125)
app//org.apache.lucene.codecs.lucene70.Lucene70DocValuesProducer$TermsDict.next(Lucene70DocValuesProducer.java:1025)
app//org.apache.lucene.index.OrdinalMap$TermsEnumIndex.next(OrdinalMap.java:59)
app//org.apache.lucene.index.OrdinalMap.<init>(OrdinalMap.java:255)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:147)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder.build(GlobalOrdinalsBuilder.java:65)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:130)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:47)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:172)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$3582/1911147663.load(Unknown Source)
app//org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:169)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:118)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals$FieldData.globalOrdinalsValues(ValuesSource.java:151)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals.globalMaxOrd(ValuesSource.java:124)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.getMaxOrd(TermsAggregatorFactory.java:214)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory$ExecutionMode$2.create(TermsAggregatorFactory.java:259)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.doCreateInternal(TermsAggregatorFactory.java:154)
app//org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory.createInternal(ValuesSourceAggregatorFactory.java:59)
app//org.elasticsearch.search.aggregations.AggregatorFactory.create(AggregatorFactory.java:216)
app//org.elasticsearch.search.aggregations.AggregatorFactories.createTopLevelAggregators(AggregatorFactories.java:217)
app//org.elasticsearch.search.aggregations.AggregationPhase.preProcess(AggregationPhase.java:55)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:112)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
app//org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
app//org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1117)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:778)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1135)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@10.0.1/java.lang.Thread.run(Thread.java:844)
2/10 snapshots sharing following 35 elements
app//org.apache.lucene.index.OrdinalMap$TermsEnumIndex.next(OrdinalMap.java:59)
app//org.apache.lucene.index.OrdinalMap.<init>(OrdinalMap.java:255)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:147)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder.build(GlobalOrdinalsBuilder.java:65)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:130)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:47)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:172)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$3582/1911147663.load(Unknown Source)
app//org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:169)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:118)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals$FieldData.globalOrdinalsValues(ValuesSource.java:151)
app//org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals.globalMaxOrd(ValuesSource.java:124)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.getMaxOrd(TermsAggregatorFactory.java:214)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory$ExecutionMode$2.create(TermsAggregatorFactory.java:259)
app//org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.doCreateInternal(TermsAggregatorFactory.java:154)
app//org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory.createInternal(ValuesSourceAggregatorFactory.java:59)
app//org.elasticsearch.search.aggregations.AggregatorFactory.create(AggregatorFactory.java:216)
app//org.elasticsearch.search.aggregations.AggregatorFactories.createTopLevelAggregators(AggregatorFactories.java:217)
app//org.elasticsearch.search.aggregations.AggregationPhase.preProcess(AggregationPhase.java:55)
app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:112)
app//org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
app//org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
app//org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
app//org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
app//org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1117)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:778)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1135)
java.base@10.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@10.0.1/java.lang.Thread.run(Thread.java:844)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment