Skip to content

Instantly share code, notes, and snippets.

@gpstathis
Created June 8, 2016 05:09
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save gpstathis/12ef17f104e3e2d7fafbae10d6eb9267 to your computer and use it in GitHub Desktop.
Save gpstathis/12ef17f104e3e2d7fafbae10d6eb9267 to your computer and use it in GitHub Desktop.
Activity stream hot threads on ES 1.4.2
::: [PROD-es-data-8][Gcg9AggXTs-L9Jt1d7uwKQ][ip-10-61-164-81][inet[/10.61.164.81:9300]]{max_local_storage_nodes=1, aws_availability_zone=us-east-1b, master=false}
101.1% (505.5ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-8][search][T#18]'
2/10 snapshots sharing following 24 elements
org.elasticsearch.index.fielddata.ordinals.SinglePackedOrdinals$Docs.getOrd(SinglePackedOrdinals.java:80)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:323)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
8/10 snapshots sharing following 27 elements
org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:42)
org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:110)
org.apache.lucene.index.MultiDocValues$OrdinalMap$2.get(MultiDocValues.java:591)
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
101.1% (505.3ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-8][search][T#70]'
7/10 snapshots sharing following 24 elements
org.elasticsearch.index.fielddata.ordinals.SinglePackedOrdinals$Docs.getOrd(SinglePackedOrdinals.java:80)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:323)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
2/10 snapshots sharing following 27 elements
org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:42)
org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:110)
org.apache.lucene.index.MultiDocValues$OrdinalMap$2.get(MultiDocValues.java:591)
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
unique snapshot
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
3.9% (19.7ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-8][management][T#2]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:734)
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1273)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
::: [PROD-es-master-1][UTysCVIvRlyHe8HeylUxLg][ip-10-153-157-121][inet[ip-10-153-157-121.ec2.internal/10.153.157.121:9300]]{max_local_storage_nodes=1, aws_availability_zone=us-east-1a, data=false, master=true}
0.9% (4.4ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-master-1][mongodb_river_slurper_unsharded:links_del][T#1]'
10/10 snapshots sharing following 23 elements
java.net.SocketInputStream.socketRead0(Native Method)
java.net.SocketInputStream.read(SocketInputStream.java:150)
java.net.SocketInputStream.read(SocketInputStream.java:121)
java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
java.io.BufferedInputStream.read1(BufferedInputStream.java:286)
java.io.BufferedInputStream.read(BufferedInputStream.java:345)
org.bson.io.Bits.readFully(Bits.java:73)
org.bson.io.Bits.readFully(Bits.java:50)
org.bson.io.Bits.readFully(Bits.java:37)
com.mongodb.Response.<init>(Response.java:42)
com.mongodb.DBPort$1.execute(DBPort.java:164)
com.mongodb.DBPort$1.execute(DBPort.java:158)
com.mongodb.DBPort.doOperation(DBPort.java:187)
com.mongodb.DBPort.call(DBPort.java:158)
com.mongodb.DBTCPConnector.innerCall(DBTCPConnector.java:290)
com.mongodb.DBTCPConnector.call(DBTCPConnector.java:269)
com.mongodb.DBTCPConnector.call(DBTCPConnector.java:235)
com.mongodb.QueryResultIterator.getMore(QueryResultIterator.java:145)
com.mongodb.QueryResultIterator.hasNext(QueryResultIterator.java:135)
com.mongodb.DBCursor._hasNext(DBCursor.java:626)
com.mongodb.DBCursor.hasNext(DBCursor.java:657)
org.elasticsearch.river.mongodb.OplogSlurper.run(OplogSlurper.java:97)
java.lang.Thread.run(Thread.java:745)
0.5% (2.2ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-master-1][transport_client_worker][T#1]{New I/O worker #1}'
10/10 snapshots sharing following 15 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
0.3% (1.7ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-master-1][transport_client_worker][T#2]{New I/O worker #2}'
10/10 snapshots sharing following 15 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
::: [PROD-es-data-9][suAld4QKSwKqjN7KMoi4-w][ip-10-16-179-189][inet[/10.16.179.189:9300]]{max_local_storage_nodes=1, aws_availability_zone=us-east-1e, master=false}
100.9% (504.6ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-9][search][T#44]'
8/10 snapshots sharing following 22 elements
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:73)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
2/10 snapshots sharing following 21 elements
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:73)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
100.9% (504.3ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-9][search][T#40]'
6/10 snapshots sharing following 24 elements
org.elasticsearch.index.fielddata.ordinals.SinglePackedOrdinals$Docs.getOrd(SinglePackedOrdinals.java:80)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:323)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:73)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
4/10 snapshots sharing following 21 elements
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:73)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
4.1% (20.3ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-9][management][T#3]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:734)
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1273)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
::: [PROD-es-data-11][PYCiSf8sR0ur7daIAkSD9g][ip-10-141-6-220][inet[/10.141.6.220:9300]]{max_local_storage_nodes=1, aws_availability_zone=us-east-1e, master=false}
101.0% (504.7ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-11][search][T#9]'
6/10 snapshots sharing following 27 elements
org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:42)
org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:110)
org.apache.lucene.index.MultiDocValues$OrdinalMap$2.get(MultiDocValues.java:591)
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
4/10 snapshots sharing following 21 elements
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
101.0% (504.7ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-11][search][T#7]'
10/10 snapshots sharing following 21 elements
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
100.9% (504.3ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-11][search][T#91]'
4/10 snapshots sharing following 27 elements
org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:42)
org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:110)
org.apache.lucene.index.MultiDocValues$OrdinalMap$2.get(MultiDocValues.java:591)
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
6/10 snapshots sharing following 24 elements
org.elasticsearch.index.fielddata.ordinals.SinglePackedOrdinals$Docs.getOrd(SinglePackedOrdinals.java:80)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:323)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
::: [PROD-es-data-2][IiHE7Z53RxqNoj9968CgPw][ip-10-31-72-19][inet[/10.31.72.19:9300]]{max_local_storage_nodes=1, aws_availability_zone=us-east-1a, master=false}
100.9% (504.4ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-2][search][T#73]'
5/10 snapshots sharing following 27 elements
org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:42)
org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:110)
org.apache.lucene.index.MultiDocValues$OrdinalMap$2.get(MultiDocValues.java:591)
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
3/10 snapshots sharing following 24 elements
org.elasticsearch.index.fielddata.ordinals.SinglePackedOrdinals$Docs.getOrd(SinglePackedOrdinals.java:80)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:323)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
2/10 snapshots sharing following 21 elements
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
100.9% (504.4ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-2][search][T#55]'
4/10 snapshots sharing following 22 elements
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:73)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
2/10 snapshots sharing following 21 elements
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:73)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
3/10 snapshots sharing following 27 elements
org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:42)
org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:110)
org.apache.lucene.index.MultiDocValues$OrdinalMap$2.get(MultiDocValues.java:591)
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
unique snapshot
org.elasticsearch.index.fielddata.ordinals.SinglePackedOrdinals$Docs.getOrd(SinglePackedOrdinals.java:80)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:323)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
100.9% (504.2ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-2][search][T#48]'
10/10 snapshots sharing following 18 elements
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
::: [PROD-es-data-1][Sqy2X-fDTWOM4be6UB73fA][ip-10-157-95-61][inet[/10.157.95.61:9300]]{max_local_storage_nodes=1, aws_availability_zone=us-east-1a, master=false}
101.0% (504.8ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-1][search][T#15]'
7/10 snapshots sharing following 21 elements
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
2/10 snapshots sharing following 27 elements
org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:42)
org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:110)
org.apache.lucene.index.MultiDocValues$OrdinalMap$2.get(MultiDocValues.java:591)
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:73)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
unique snapshot
org.elasticsearch.index.fielddata.ordinals.SinglePackedOrdinals$Docs.getOrd(SinglePackedOrdinals.java:80)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:323)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:73)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
100.9% (504.5ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-1][search][T#44]'
8/10 snapshots sharing following 27 elements
org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:42)
org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:110)
org.apache.lucene.index.MultiDocValues$OrdinalMap$2.get(MultiDocValues.java:591)
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
2/10 snapshots sharing following 24 elements
org.elasticsearch.index.fielddata.ordinals.SinglePackedOrdinals$Docs.getOrd(SinglePackedOrdinals.java:80)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:323)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
6.3% (31.6ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-1][management][T#1]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:734)
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1273)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
::: [PROD-es-data-3][5obFOPPKRjORiSlmhLAvLA][ip-10-184-173-160][inet[/10.184.173.160:9300]]{max_local_storage_nodes=1, aws_availability_zone=us-east-1b, master=false}
100.8% (504.2ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-3][search][T#80]'
10/10 snapshots sharing following 27 elements
org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:42)
org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:110)
org.apache.lucene.index.MultiDocValues$OrdinalMap$2.get(MultiDocValues.java:591)
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
100.8% (504.1ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-3][search][T#4]'
8/10 snapshots sharing following 27 elements
org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:42)
org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:110)
org.apache.lucene.index.MultiDocValues$OrdinalMap$2.get(MultiDocValues.java:591)
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:73)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
2/10 snapshots sharing following 21 elements
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:73)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
4.1% (20.6ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-3][management][T#5]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:734)
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1273)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
::: [PROD-es-data-6][swUyV_YFRmG3w62I8Simpg][ip-10-123-135-97][inet[/10.123.135.97:9300]]{max_local_storage_nodes=1, aws_availability_zone=us-east-1e, master=false}
100.0% (499.8ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-6][search][T#79]'
4/10 snapshots sharing following 27 elements
org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:42)
org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:110)
org.apache.lucene.index.MultiDocValues$OrdinalMap$2.get(MultiDocValues.java:591)
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
5/10 snapshots sharing following 24 elements
org.elasticsearch.index.fielddata.ordinals.SinglePackedOrdinals$Docs.getOrd(SinglePackedOrdinals.java:80)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:323)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
unique snapshot
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
3.8% (18.8ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-6][management][T#5]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:734)
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1273)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
3.7% (18.7ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-6][management][T#3]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:734)
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1273)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
::: [PROD-es-data-10][Gn-eXux4R76pfysexsOemw][ip-10-97-137-166][inet[/10.97.137.166:9300]]{max_local_storage_nodes=1, aws_availability_zone=us-east-1a, master=false}
101.0% (505.1ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-10][search][T#76]'
3/10 snapshots sharing following 27 elements
org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:42)
org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:110)
org.apache.lucene.index.MultiDocValues$OrdinalMap$2.get(MultiDocValues.java:591)
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
7/10 snapshots sharing following 24 elements
org.elasticsearch.index.fielddata.ordinals.SinglePackedOrdinals$Docs.getOrd(SinglePackedOrdinals.java:80)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:323)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
101.0% (504.8ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-10][search][T#16]'
4/10 snapshots sharing following 24 elements
org.elasticsearch.index.fielddata.ordinals.SinglePackedOrdinals$Docs.getOrd(SinglePackedOrdinals.java:80)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:323)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
5/10 snapshots sharing following 27 elements
org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:42)
org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:110)
org.apache.lucene.index.MultiDocValues$OrdinalMap$2.get(MultiDocValues.java:591)
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
unique snapshot
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
6.6% (32.9ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-10][management][T#2]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:734)
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1273)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
::: [PROD-es-data-5][mld4Gc8SSmOYgAKVCGheow][ip-10-13-199-229][inet[/10.13.199.229:9300]]{max_local_storage_nodes=1, aws_availability_zone=us-east-1e, master=false}
101.0% (505ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-5][search][T#3]'
6/10 snapshots sharing following 27 elements
org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:42)
org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:110)
org.apache.lucene.index.MultiDocValues$OrdinalMap$2.get(MultiDocValues.java:591)
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
3/10 snapshots sharing following 24 elements
org.elasticsearch.index.fielddata.ordinals.SinglePackedOrdinals$Docs.getOrd(SinglePackedOrdinals.java:80)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:323)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
unique snapshot
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
101.0% (504.8ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-5][search][T#28]'
4/10 snapshots sharing following 24 elements
org.elasticsearch.index.fielddata.ordinals.SinglePackedOrdinals$Docs.getOrd(SinglePackedOrdinals.java:80)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:323)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
6/10 snapshots sharing following 21 elements
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
3.6% (18.1ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-5][management][T#3]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:734)
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1273)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
::: [PROD-es-data-12][bThjC7ctSv-eCG8Bdv_xdA][ip-10-102-139-240][inet[/10.102.139.240:9300]]{max_local_storage_nodes=1, aws_availability_zone=us-east-1b, master=false}
100.9% (504.6ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-12][search][T#86]'
10/10 snapshots sharing following 22 elements
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
100.9% (504.3ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-12][search][T#32]'
6/10 snapshots sharing following 24 elements
org.elasticsearch.index.fielddata.ordinals.SinglePackedOrdinals$Docs.getOrd(SinglePackedOrdinals.java:80)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:323)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
2/10 snapshots sharing following 27 elements
org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:42)
org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:110)
org.apache.lucene.index.MultiDocValues$OrdinalMap$2.get(MultiDocValues.java:591)
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
2/10 snapshots sharing following 21 elements
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
6.6% (32.7ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-12][management][T#5]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:734)
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1273)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
::: [PROD-es-data-7][1LzuJzjsQHedRE6Dndee_w][ip-10-41-194-71][inet[/10.41.194.71:9300]]{max_local_storage_nodes=1, aws_availability_zone=us-east-1a, master=false}
3.8% (18.7ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-7][management][T#2]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:734)
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1273)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
3.7% (18.6ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-7][management][T#5]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:734)
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1273)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
3.7% (18.4ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-7][management][T#4]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:734)
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1273)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
::: [PROD-es-data-4][LK19xUlySGy4RMn_Frd5Qw][ip-10-140-168-42][inet[/10.140.168.42:9300]]{max_local_storage_nodes=1, aws_availability_zone=us-east-1b, master=false}
100.9% (504.3ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-4][search][T#9]'
4/10 snapshots sharing following 24 elements
org.elasticsearch.index.fielddata.ordinals.SinglePackedOrdinals$Docs.getOrd(SinglePackedOrdinals.java:80)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:323)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
6/10 snapshots sharing following 24 elements
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
100.9% (504.3ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-4][search][T#30]'
2/10 snapshots sharing following 24 elements
org.elasticsearch.index.fielddata.ordinals.SinglePackedOrdinals$Docs.getOrd(SinglePackedOrdinals.java:80)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:323)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
7/10 snapshots sharing following 27 elements
org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:42)
org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:110)
org.apache.lucene.index.MultiDocValues$OrdinalMap$2.get(MultiDocValues.java:591)
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
unique snapshot
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
100.8% (503.9ms out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-data-4][search][T#10]'
5/10 snapshots sharing following 27 elements
org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:42)
org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:110)
org.apache.lucene.index.MultiDocValues$OrdinalMap$2.get(MultiDocValues.java:591)
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
4/10 snapshots sharing following 24 elements
org.elasticsearch.index.fielddata.ordinals.SinglePackedOrdinals$Docs.getOrd(SinglePackedOrdinals.java:80)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:323)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
unique snapshot
org.apache.lucene.util.LongValues.get(LongValues.java:45)
org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData$1.getOrd(ParentChildIndexFieldData.java:326)
org.elasticsearch.index.search.child.ParentConstantScoreQuery$ChildrenDocIdIterator.match(ParentConstantScoreQuery.java:225)
org.apache.lucene.search.FilteredDocIdSetIterator.nextDoc(FilteredDocIdSetIterator.java:60)
org.elasticsearch.index.search.child.ConstantScorer.nextDoc(ConstantScorer.java:48)
org.elasticsearch.common.lucene.docset.DocIdSets.toCacheable(DocIdSets.java:84)
org.elasticsearch.index.search.child.CustomQueryWrappingFilter.getDocIdSet(CustomQueryWrappingFilter.java:74)
org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:83)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:46)
org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:157)
org.apache.lucene.search.FilteredQuery$RandomAccessFilterStrategy.filteredScorer(FilteredQuery.java:542)
org.apache.lucene.search.FilteredQuery$FilterStrategy.filteredBulkScorer(FilteredQuery.java:504)
org.apache.lucene.search.FilteredQuery$1.bulkScorer(FilteredQuery.java:150)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:117)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:275)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:275)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
::: [PROD-es-master-3][YRDyx5PJSA2gbdGETNtZEw][ip-10-147-6-45][inet[/10.147.6.45:9300]]{max_local_storage_nodes=1, aws_availability_zone=us-east-1e, data=false, master=true}
0.1% (417.4micros out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-master-3][[transport_server_worker.default]][T#2]{New I/O worker #5}'
10/10 snapshots sharing following 15 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
0.0% (111.9micros out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-master-3][[transport_server_worker.default]][T#1]{New I/O worker #4}'
10/10 snapshots sharing following 15 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
0.0% (106.9micros out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-master-3][listener][T#1]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:737)
java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1265)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
::: [PROD-es-master-2][JMqVfUB8TNKdJ2tuSSzw9A][ip-10-142-165-204][inet[/10.142.165.204:9300]]{max_local_storage_nodes=1, aws_availability_zone=us-east-1b, data=false, master=true}
0.0% (216.1micros out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-master-2][[transport_server_worker.default]][T#2]{New I/O worker #5}'
10/10 snapshots sharing following 15 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
0.0% (122micros out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-master-2][scheduler][T#1]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
0.0% (114.5micros out of 500ms) cpu usage by thread 'elasticsearch[PROD-es-master-2][[transport_server_worker.default]][T#1]{New I/O worker #4}'
10/10 snapshots sharing following 15 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment