Skip to content

Instantly share code, notes, and snippets.

@sns-seb

sns-seb/gist:6239785785ec6972cef5b2b7acbda939

Created January 31, 2020 14:38
Show Gist options
  • Save sns-seb/6239785785ec6972cef5b2b7acbda939 to your computer and use it in GitHub Desktop.
Save sns-seb/6239785785ec6972cef5b2b7acbda939 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
gistfile1.txt
[slesaint@search-n2-Dev SC-1226]$ curl -X GET "10.0.72.231:9032/_nodes/hot_threads?pretty"
::: {search-n3-Dev}{0NZ7ydJgTq6Cz7oO9-bLfw}{dvHu0TTVRZy8_U7dv131OQ}{10.0.95.194}{10.0.95.194:9001}{rack_id=search-n3-Dev}
Hot threads at 2020-01-31T14:36:48.958Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
::: {search-n2-Dev}{eCDgBprbSCGEZvNMGLO-aw}{7aMbRrj5SiipXNTGseXRlQ}{10.0.72.231}{10.0.72.231:9001}{rack_id=search-n2-Dev}
Hot threads at 2020-01-31T14:36:48.960Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
::: {search-n1-Dev}{qGLbuap-QT-Zt9GmAD7Hzg}{yJL22PSuQJWoo0HSJo3qzA}{10.0.60.168}{10.0.60.168:9001}{rack_id=search-n1-Dev}
Hot threads at 2020-01-31T14:36:48.958Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
[slesaint@search-n2-Dev SC-1226]$ curl -X GET "10.0.72.231:9032/_nodes/hot_threads?pretty"
::: {search-n3-Dev}{0NZ7ydJgTq6Cz7oO9-bLfw}{dvHu0TTVRZy8_U7dv131OQ}{10.0.95.194}{10.0.95.194:9001}{rack_id=search-n3-Dev}
Hot threads at 2020-01-31T14:36:53.657Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
::: {search-n2-Dev}{eCDgBprbSCGEZvNMGLO-aw}{7aMbRrj5SiipXNTGseXRlQ}{10.0.72.231}{10.0.72.231:9001}{rack_id=search-n2-Dev}
Hot threads at 2020-01-31T14:36:53.664Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
81.9% (409.6ms out of 500ms) cpu usage by thread 'elasticsearch[search-n2-Dev][search][T#1]'
10/10 snapshots sharing following 27 elements
org.elasticsearch.common.lucene.Lucene$3.get(Lucene.java:824)
org.elasticsearch.search.aggregations.bucket.filter.FilterAggregator$1.collect(FilterAggregator.java:65)
org.elasticsearch.search.aggregations.bucket.BucketsAggregator.collectExistingBucket(BucketsAggregator.java:84)
org.elasticsearch.search.aggregations.bucket.BucketsAggregator.collectBucket(BucketsAggregator.java:76)
org.elasticsearch.search.aggregations.bucket.global.GlobalAggregator$1.collect(GlobalAggregator.java:49)
org.elasticsearch.search.aggregations.MultiBucketCollector$MultiLeafBucketCollector.collect(MultiBucketCollector.java:195)
org.elasticsearch.search.aggregations.LeafBucketCollector.collect(LeafBucketCollector.java:82)
org.apache.lucene.search.MatchAllDocsQuery$1$1.score(MatchAllDocsQuery.java:62)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:670)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
org.elasticsearch.search.aggregations.AggregationPhase.execute(AggregationPhase.java:117)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:120)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1086)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
64.8% (323.9ms out of 500ms) cpu usage by thread 'elasticsearch[search-n2-Dev][search][T#3]'
10/10 snapshots sharing following 22 elements
org.elasticsearch.search.aggregations.MultiBucketCollector$MultiLeafBucketCollector.collect(MultiBucketCollector.java:204)
org.elasticsearch.search.aggregations.LeafBucketCollector.collect(LeafBucketCollector.java:82)
org.apache.lucene.search.MatchAllDocsQuery$1$1.score(MatchAllDocsQuery.java:62)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:670)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
org.elasticsearch.search.aggregations.AggregationPhase.execute(AggregationPhase.java:117)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:120)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1086)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
56.4% (281.7ms out of 500ms) cpu usage by thread 'elasticsearch[search-n2-Dev][search][T#2]'
10/10 snapshots sharing following 22 elements
org.elasticsearch.search.aggregations.MultiBucketCollector$MultiLeafBucketCollector.collect(MultiBucketCollector.java:204)
org.elasticsearch.search.aggregations.LeafBucketCollector.collect(LeafBucketCollector.java:82)
org.apache.lucene.search.MatchAllDocsQuery$1$1.score(MatchAllDocsQuery.java:62)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:670)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
org.elasticsearch.search.aggregations.AggregationPhase.execute(AggregationPhase.java:117)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:120)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1086)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
::: {search-n1-Dev}{qGLbuap-QT-Zt9GmAD7Hzg}{yJL22PSuQJWoo0HSJo3qzA}{10.0.60.168}{10.0.60.168:9001}{rack_id=search-n1-Dev}
Hot threads at 2020-01-31T14:36:53.659Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
100.2% (500.7ms out of 500ms) cpu usage by thread 'elasticsearch[search-n1-Dev][search][T#1]'
9/10 snapshots sharing following 25 elements
org.elasticsearch.search.aggregations.bucket.BucketsAggregator.collectExistingBucket(BucketsAggregator.java:84)
org.elasticsearch.search.aggregations.bucket.BucketsAggregator.collectBucket(BucketsAggregator.java:76)
org.elasticsearch.search.aggregations.bucket.global.GlobalAggregator$1.collect(GlobalAggregator.java:49)
org.elasticsearch.search.aggregations.MultiBucketCollector$MultiLeafBucketCollector.collect(MultiBucketCollector.java:195)
org.elasticsearch.search.aggregations.LeafBucketCollector.collect(LeafBucketCollector.java:82)
org.apache.lucene.search.MatchAllDocsQuery$1$1.score(MatchAllDocsQuery.java:62)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:670)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
org.elasticsearch.search.aggregations.AggregationPhase.execute(AggregationPhase.java:117)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:120)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1086)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
unique snapshot
org.elasticsearch.search.aggregations.MultiBucketCollector$MultiLeafBucketCollector.collect(MultiBucketCollector.java:204)
org.elasticsearch.search.aggregations.LeafBucketCollector.collect(LeafBucketCollector.java:82)
org.apache.lucene.search.MatchAllDocsQuery$1$1.score(MatchAllDocsQuery.java:62)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:670)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
org.elasticsearch.search.aggregations.AggregationPhase.execute(AggregationPhase.java:117)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:120)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1086)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
99.8% (499ms out of 500ms) cpu usage by thread 'elasticsearch[search-n1-Dev][search][T#2]'
10/10 snapshots sharing following 25 elements
org.elasticsearch.search.aggregations.bucket.BucketsAggregator.collectExistingBucket(BucketsAggregator.java:84)
org.elasticsearch.search.aggregations.bucket.BucketsAggregator.collectBucket(BucketsAggregator.java:76)
org.elasticsearch.search.aggregations.bucket.global.GlobalAggregator$1.collect(GlobalAggregator.java:49)
org.elasticsearch.search.aggregations.MultiBucketCollector$MultiLeafBucketCollector.collect(MultiBucketCollector.java:195)
org.elasticsearch.search.aggregations.LeafBucketCollector.collect(LeafBucketCollector.java:82)
org.apache.lucene.search.MatchAllDocsQuery$1$1.score(MatchAllDocsQuery.java:62)
org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:670)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
org.elasticsearch.search.aggregations.AggregationPhase.execute(AggregationPhase.java:117)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:120)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1086)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment