elastic search hot thread
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
::: [es-cluster][iTnWLzezQi2QUhFI2bmxBw][es-cluster.xxx][inet[/10.0.18.8:9300]]{master=true} | |
Hot threads at 2015-11-23T16:45:07.894Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
33.5% (167.5ms out of 500ms) cpu usage by thread 'elasticsearch[es-cluster][search][T#9]' | |
2/10 snapshots sharing following 3 elements | |
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
7/10 snapshots sharing following 10 elements | |
sun.misc.Unsafe.park(Native Method) | |
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) | |
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735) | |
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644) | |
java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137) | |
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162) | |
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) | |
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
30.8% (153.9ms out of 500ms) cpu usage by thread 'elasticsearch[es-cluster][search][T#10]' | |
9/10 snapshots sharing following 2 elements | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
::: [mb-es-clusterpg123a1n4][2rTRpq2cTtWANfWTrG7jEg][mb-es-clusterpg123a1n4.xxx][inet[/10.0.18.7:9300]]{master=true} | |
Hot threads at 2015-11-23T16:45:07.900Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
40.6% (203ms out of 500ms) cpu usage by thread 'elasticsearch[mb-es-clusterpg123a1n4][search][T#7]' | |
6/10 snapshots sharing following 2 elements | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
38.1% (190.2ms out of 500ms) cpu usage by thread 'elasticsearch[mb-es-clusterpg123a1n4][search][T#13]' | |
2/10 snapshots sharing following 3 elements | |
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
5/10 snapshots sharing following 10 elements | |
sun.misc.Unsafe.park(Native Method) | |
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) | |
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735) | |
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644) | |
java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137) | |
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162) | |
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) | |
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
::: [es-cluster4][u9bAM0WESMOADtvpOMfnVA][es-cluster4.xxx][inet[/10.0.18.174:9300]]{master=true} | |
Hot threads at 2015-11-23T16:45:07.896Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
13.2% (66ms out of 500ms) cpu usage by thread 'elasticsearch[es-cluster4][search][T#8]' | |
8/10 snapshots sharing following 2 elements | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
12.5% (62.3ms out of 500ms) cpu usage by thread 'elasticsearch[es-cluster4][search][T#7]' | |
2/10 snapshots sharing following 22 elements | |
org.elasticsearch.common.lucene.docset.AndDocIdSet$AndBits.get(AndDocIdSet.java:116) | |
org.apache.lucene.search.FilteredDocIdSet$1.get(FilteredDocIdSet.java:72) | |
org.apache.lucene.search.FilteredQuery$QueryFirstScorer.nextDoc(FilteredQuery.java:177) | |
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:192) | |
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:163) | |
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:35) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:621) | |
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:491) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:448) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:281) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:269) | |
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:157) | |
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:301) | |
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:312) | |
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776) | |
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767) | |
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:279) | |
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:36) | |
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
5/10 snapshots sharing following 10 elements | |
sun.misc.Unsafe.park(Native Method) | |
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) | |
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735) | |
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644) | |
java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137) | |
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162) | |
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) | |
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
::: [es-cluster2][OGjEOQ76S-Oo-mlUITB2vA][es-cluster2.xxx][inet[/10.0.18.172:9300]]{master=true} | |
Hot threads at 2015-11-23T16:45:07.896Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
24.4% (121.8ms out of 500ms) cpu usage by thread 'elasticsearch[es-cluster2][search][T#3]' | |
2/10 snapshots sharing following 12 elements | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:281) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:269) | |
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:157) | |
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:301) | |
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:312) | |
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776) | |
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767) | |
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:279) | |
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:36) | |
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
7/10 snapshots sharing following 10 elements | |
sun.misc.Unsafe.park(Native Method) | |
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) | |
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735) | |
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644) | |
java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137) | |
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162) | |
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) | |
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
20.9% (104.3ms out of 500ms) cpu usage by thread 'elasticsearch[es-cluster2][search][T#9]' | |
5/10 snapshots sharing following 2 elements | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
20.5% (102.7ms out of 500ms) cpu usage by thread 'elasticsearch[es-cluster2][search][T#2]' | |
5/10 snapshots sharing following 2 elements | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
::: [es-cluster5][JPzUx2rDRQWgKS8aHa5txQ][es-cluster5.xxx][inet[/10.0.18.175:9300]]{master=true} | |
Hot threads at 2015-11-23T16:45:07.893Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
16.8% (84.1ms out of 500ms) cpu usage by thread 'elasticsearch[es-cluster5][search][T#1]' | |
6/10 snapshots sharing following 2 elements | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
16.6% (82.8ms out of 500ms) cpu usage by thread 'elasticsearch[es-cluster5][search][T#7]' | |
10/10 snapshots sharing following 2 elements | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
14.7% (73.3ms out of 500ms) cpu usage by thread 'elasticsearch[es-cluster5][search][T#8]' | |
6/10 snapshots sharing following 2 elements | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
::: [es-cluster3][S8VenWxgQoCBfjhLeVTJzQ][es-cluster3.xxx][inet[/10.0.18.173:9300]]{master=true} | |
Hot threads at 2015-11-23T16:45:07.895Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
26.9% (134.5ms out of 500ms) cpu usage by thread 'elasticsearch[es-cluster3][search][T#1]' | |
5/10 snapshots sharing following 2 elements | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
25.8% (128.7ms out of 500ms) cpu usage by thread 'elasticsearch[es-cluster3][search][T#3]' | |
unique snapshot | |
org.apache.lucene.search.ReqExclScorer.nextDoc(ReqExclScorer.java:50) | |
org.elasticsearch.common.lucene.search.function.CustomBoostFactorScorer$AnyNextDoc.nextDoc(CustomBoostFactorScorer.java:128) | |
org.elasticsearch.common.lucene.search.function.CustomBoostFactorScorer.nextDoc(CustomBoostFactorScorer.java:64) | |
org.apache.lucene.search.FilteredQuery$QueryFirstScorer.nextDoc(FilteredQuery.java:176) | |
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:192) | |
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:163) | |
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:35) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:621) | |
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:491) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:448) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:281) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:269) | |
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:157) | |
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:301) | |
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:312) | |
org.elasticsearch.search.action.SearchServiceTransportAction$5.call(SearchServiceTransportAction.java:231) | |
org.elasticsearch.search.action.SearchServiceTransportAction$5.call(SearchServiceTransportAction.java:228) | |
org.elasticsearch.search.action.SearchServiceTransportAction$23.run(SearchServiceTransportAction.java:559) | |
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
21.0% (104.7ms out of 500ms) cpu usage by thread 'elasticsearch[es-cluster3][search][T#10]' | |
2/10 snapshots sharing following 3 elements | |
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
7/10 snapshots sharing following 10 elements | |
sun.misc.Unsafe.park(Native Method) | |
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) | |
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735) | |
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644) | |
java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137) | |
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162) | |
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) | |
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
::: [es-cluster1][1wiw9MxrRva84OfApHrYew][es-cluster1.xxx][inet[/10.0.18.171:9300]]{master=true} | |
Hot threads at 2015-11-23T16:45:07.896Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
23.4% (116.8ms out of 500ms) cpu usage by thread 'elasticsearch[es-cluster1][search][T#7]' | |
2/10 snapshots sharing following 19 elements | |
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:192) | |
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:163) | |
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:35) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:621) | |
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:491) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:448) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:281) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:269) | |
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:157) | |
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:301) | |
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:312) | |
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776) | |
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767) | |
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:279) | |
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:36) | |
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
3/10 snapshots sharing following 10 elements | |
sun.misc.Unsafe.park(Native Method) | |
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) | |
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735) | |
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644) | |
java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137) | |
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162) | |
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) | |
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
18.6% (92.7ms out of 500ms) cpu usage by thread 'elasticsearch[es-cluster1][search][T#4]' | |
2/10 snapshots sharing following 20 elements | |
org.apache.lucene.search.FilteredQuery$QueryFirstScorer.nextDoc(FilteredQuery.java:177) | |
org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:192) | |
org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:163) | |
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:35) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:621) | |
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:491) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:448) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:281) | |
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:269) | |
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:157) | |
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:301) | |
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:312) | |
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776) | |
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767) | |
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:279) | |
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:36) | |
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) | |
5/10 snapshots sharing following 10 elements | |
sun.misc.Unsafe.park(Native Method) | |
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) | |
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735) | |
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644) | |
java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137) | |
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162) | |
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) | |
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) | |
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
java.lang.Thread.run(Thread.java:745) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment