Skip to content

Instantly share code, notes, and snippets.

@pavanladi98

pavanladi98/hot_threads _with_search_requests Secret

Created September 20, 2020 08:18
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save pavanladi98/dc5cab94be43dc2771ef8362bccc4a7a to your computer and use it in GitHub Desktop.
Save pavanladi98/dc5cab94be43dc2771ef8362bccc4a7a to your computer and use it in GitHub Desktop.
Download ZIP
ES 7.4 hot threads with search requests load
Raw
hot_threads _with_search_requests
::: {0021a8ed3db7c178baa9f7a80a43c184}{mmemg5PER6ShZ739q4C6Bw}{z0ZvG3rIQom2pmkgy6EGKA}{x.x.x.x}{x.x.x.x:9300}{di}{zone=us-east-1b, distributed_snapshot_deletion_enabled=true}
Hot threads at 2020-09-20T08:01:22.831Z, interval=500ms, busiestThreads=10, ignoreIdleThreads=false:
100.2% (500.7ms out of 500ms) cpu usage by thread 'elasticsearch[0021a8ed3db7c178baa9f7a80a43c184][warmer][T#3]'
6/10 snapshots sharing following 20 elements
app//org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:202)
app//org.apache.lucene.index.OrdinalMap.<init>(OrdinalMap.java:261)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:147)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder.build(GlobalOrdinalsBuilder.java:65)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:142)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:48)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:172)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$3183/0x0000000801845440.load(Unknown Source)
app//org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:169)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobalInternal(SortedSetDVOrdinalsIndexFieldData.java:130)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:96)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:48)
app//org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:135)
app//org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$3097/0x00000008017ed440.run(Unknown Source)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:712)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
4/10 snapshots sharing following 18 elements
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:147)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder.build(GlobalOrdinalsBuilder.java:65)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:142)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:48)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:172)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$3183/0x0000000801845440.load(Unknown Source)
app//org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:169)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobalInternal(SortedSetDVOrdinalsIndexFieldData.java:130)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:96)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:48)
app//org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:135)
app//org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$3097/0x00000008017ed440.run(Unknown Source)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:712)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
100.2% (500.7ms out of 500ms) cpu usage by thread 'elasticsearch[0021a8ed3db7c178baa9f7a80a43c184][warmer][T#1]'
2/10 snapshots sharing following 21 elements
app//org.apache.lucene.util.PriorityQueue.downHeap(PriorityQueue.java:275)
app//org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:202)
app//org.apache.lucene.index.OrdinalMap.<init>(OrdinalMap.java:261)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:147)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder.build(GlobalOrdinalsBuilder.java:65)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:142)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:48)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:172)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$3183/0x0000000801845440.load(Unknown Source)
app//org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:169)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobalInternal(SortedSetDVOrdinalsIndexFieldData.java:130)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:96)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:48)
app//org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:135)
app//org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$3097/0x00000008017ed440.run(Unknown Source)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:712)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
3/10 snapshots sharing following 18 elements
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:147)
app//org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder.build(GlobalOrdinalsBuilder.java:65)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:142)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:48)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:172)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$3183/0x0000000801845440.load(Unknown Source)
app//org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
app//org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:169)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobalInternal(SortedSetDVOrdinalsIndexFieldData.java:130)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:96)
app//org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:48)
app//org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:135)
app//org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$3097/0x00000008017ed440.run(Unknown Source)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:712)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
5/10 snapshots sharing following 9 elements
java.base@11.0.6/jdk.internal.misc.Unsafe.park(Native Method)
java.base@11.0.6/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
java.base@11.0.6/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:740)
java.base@11.0.6/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684)
java.base@11.0.6/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1374)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
17.0% (85ms out of 500ms) cpu usage by thread 'elasticsearch[0021a8ed3db7c178baa9f7a80a43c184][transport_worker][T#6]'
10/10 snapshots sharing following 9 elements
java.base@11.0.6/sun.nio.ch.EPoll.wait(Native Method)
java.base@11.0.6/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
java.base@11.0.6/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
java.base@11.0.6/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136)
io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:806)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:454)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:918)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
0.3% (1.6ms out of 500ms) cpu usage by thread 'qtp173708546-4826'
10/10 snapshots sharing following 8 elements
java.base@11.0.6/jdk.internal.misc.Unsafe.park(Native Method)
java.base@11.0.6/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
java.base@11.0.6/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
app//org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
0.2% (1ms out of 500ms) cpu usage by thread 'elasticsearch[0021a8ed3db7c178baa9f7a80a43c184][transport_worker][T#8]'
10/10 snapshots sharing following 9 elements
java.base@11.0.6/sun.nio.ch.EPoll.wait(Native Method)
java.base@11.0.6/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
java.base@11.0.6/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
java.base@11.0.6/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136)
io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:806)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:454)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:918)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
0.2% (868.9micros out of 500ms) cpu usage by thread 'elasticsearch[0021a8ed3db7c178baa9f7a80a43c184][management][T#3]'
10/10 snapshots sharing following 9 elements
java.base@11.0.6/jdk.internal.misc.Unsafe.park(Native Method)
java.base@11.0.6/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
java.base@11.0.6/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:740)
java.base@11.0.6/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684)
java.base@11.0.6/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1374)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
0.1% (402.7micros out of 500ms) cpu usage by thread 'qtp173708546-4829'
10/10 snapshots sharing following 8 elements
java.base@11.0.6/jdk.internal.misc.Unsafe.park(Native Method)
java.base@11.0.6/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
java.base@11.0.6/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
app//org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
0.1% (338.7micros out of 500ms) cpu usage by thread 'elasticsearch[0021a8ed3db7c178baa9f7a80a43c184][scheduler][T#1]'
10/10 snapshots sharing following 9 elements
java.base@11.0.6/jdk.internal.misc.Unsafe.park(Native Method)
java.base@11.0.6/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
java.base@11.0.6/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
java.base@11.0.6/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
java.base@11.0.6/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
java.base@11.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
0.0% (209.9micros out of 500ms) cpu usage by thread 'elasticsearch[0021a8ed3db7c178baa9f7a80a43c184][transport_worker][T#4]'
10/10 snapshots sharing following 9 elements
java.base@11.0.6/sun.nio.ch.EPoll.wait(Native Method)
java.base@11.0.6/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
java.base@11.0.6/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
java.base@11.0.6/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136)
io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:806)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:454)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:918)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
0.0% (169.3micros out of 500ms) cpu usage by thread 'qtp173708546-111 Selector0'
10/10 snapshots sharing following 9 elements
java.base@11.0.6/sun.nio.ch.EPoll.wait(Native Method)
java.base@11.0.6/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
java.base@11.0.6/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
java.base@11.0.6/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136)
app//org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
app//org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
app//org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
java.base@11.0.6/java.lang.Thread.run(Thread.java:834)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment