Skip to content

Instantly share code, notes, and snippets.

@kaidence
Created March 27, 2014 19:35
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save kaidence/2b95c207f4e6a79841c5 to your computer and use it in GitHub Desktop.
Save kaidence/2b95c207f4e6a79841c5 to your computer and use it in GitHub Desktop.
ElasticSearch eating all CPU
::: [dev4][mlUqJr_XROyXARQO0PjP7A][dev4.address][inet[10.1.14.19/10.1.14.19:9300]]
63.2% (6.3s out of 10s) cpu usage by thread 'elasticsearch[dev4][search][T#43]'
10/10 snapshots sharing following 17 elements
java.util.HashMap.getEntry(HashMap.java:469)
java.util.HashMap.get(HashMap.java:421)
org.elasticsearch.search.scan.ScanContext$ScanFilter.getDocIdSet(ScanContext.java:156)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:45)
org.apache.lucene.search.FilteredQuery$1.scorer(FilteredQuery.java:128)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:173)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.scan.ScanContext.execute(ScanContext.java:52)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
org.elasticsearch.search.SearchService.executeScan(SearchService.java:225)
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:474)
org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.executePhase(TransportSearchScrollScanAction.java:210)
org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction$2.run(TransportSearchScrollScanAction.java:180)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
60.5% (6s out of 10s) cpu usage by thread 'elasticsearch[dev4][search][T#14]'
10/10 snapshots sharing following 17 elements
java.util.HashMap.getEntry(HashMap.java:469)
java.util.HashMap.get(HashMap.java:421)
org.elasticsearch.search.scan.ScanContext$ScanFilter.getDocIdSet(ScanContext.java:156)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:45)
org.apache.lucene.search.FilteredQuery$1.scorer(FilteredQuery.java:128)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:173)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.scan.ScanContext.execute(ScanContext.java:52)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
org.elasticsearch.search.SearchService.executeScan(SearchService.java:225)
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:474)
org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.executePhase(TransportSearchScrollScanAction.java:210)
org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction$2.run(TransportSearchScrollScanAction.java:180)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
58.9% (5.8s out of 10s) cpu usage by thread 'elasticsearch[dev4][search][T#7]'
10/10 snapshots sharing following 17 elements
java.util.HashMap.getEntry(HashMap.java:469)
java.util.HashMap.get(HashMap.java:421)
org.elasticsearch.search.scan.ScanContext$ScanFilter.getDocIdSet(ScanContext.java:156)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:45)
org.apache.lucene.search.FilteredQuery$1.scorer(FilteredQuery.java:128)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:173)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.scan.ScanContext.execute(ScanContext.java:52)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
org.elasticsearch.search.SearchService.executeScan(SearchService.java:225)
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:474)
org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.executePhase(TransportSearchScrollScanAction.java:210)
org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction$2.run(TransportSearchScrollScanAction.java:180)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
56.7% (5.6s out of 10s) cpu usage by thread 'elasticsearch[dev4][search][T#8]'
10/10 snapshots sharing following 17 elements
java.util.HashMap.getEntry(HashMap.java:469)
java.util.HashMap.get(HashMap.java:421)
org.elasticsearch.search.scan.ScanContext$ScanFilter.getDocIdSet(ScanContext.java:156)
org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter.getDocIdSet(ApplyAcceptedDocsFilter.java:45)
org.apache.lucene.search.FilteredQuery$1.scorer(FilteredQuery.java:128)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:173)
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:309)
org.elasticsearch.search.scan.ScanContext.execute(ScanContext.java:52)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
org.elasticsearch.search.SearchService.executeScan(SearchService.java:225)
org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:474)
org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.executePhase(TransportSearchScrollScanAction.java:210)
org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction$2.run(TransportSearchScrollScanAction.java:180)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
4.6% (462.7ms out of 10s) cpu usage by thread 'elasticsearch[dev4][refresh][T#2]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
4.1% (410.9ms out of 10s) cpu usage by thread 'elasticsearch[dev4][index][T#13]'
10/10 snapshots sharing following 2 elements
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
3.9% (390.1ms out of 10s) cpu usage by thread 'elasticsearch[dev4][index][T#24]'
10/10 snapshots sharing following 10 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:706)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.take(LinkedTransferQueue.java:1109)
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
3.8% (378.3ms out of 10s) cpu usage by thread 'elasticsearch[dev4][index][T#7]'
8/10 snapshots sharing following 10 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:706)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.take(LinkedTransferQueue.java:1109)
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
2/10 snapshots sharing following 25 elements
java.lang.System.identityHashCode(Native Method)
org.apache.lucene.util.WeakIdentityMap$IdentityWeakReference.<init>(WeakIdentityMap.java:250)
org.apache.lucene.util.WeakIdentityMap.put(WeakIdentityMap.java:131)
org.apache.lucene.store.ByteBufferIndexInput.buildSlice(ByteBufferIndexInput.java:236)
org.apache.lucene.store.ByteBufferIndexInput.clone(ByteBufferIndexInput.java:187)
org.apache.lucene.store.ByteBufferIndexInput.clone(ByteBufferIndexInput.java:39)
org.apache.lucene.store.Directory$SlicedIndexInput.clone(Directory.java:288)
org.apache.lucene.store.Directory$SlicedIndexInput.clone(Directory.java:269)
org.apache.lucene.codecs.BlockTreeTermsReader$FieldReader$SegmentTermsEnum.initIndexInput(BlockTreeTermsReader.java:1324)
org.apache.lucene.codecs.BlockTreeTermsReader$FieldReader$SegmentTermsEnum$Frame.loadBlock(BlockTreeTermsReader.java:2357)
org.apache.lucene.codecs.BlockTreeTermsReader$FieldReader$SegmentTermsEnum.seekExact(BlockTreeTermsReader.java:1703)
org.elasticsearch.common.lucene.uid.Versions.loadDocIdAndVersion(Versions.java:94)
org.elasticsearch.common.lucene.uid.Versions.loadDocIdAndVersion(Versions.java:65)
org.elasticsearch.common.lucene.uid.Versions.loadVersion(Versions.java:82)
org.elasticsearch.index.engine.internal.InternalEngine.loadCurrentVersionFromIndex(InternalEngine.java:1332)
org.elasticsearch.index.engine.internal.InternalEngine.innerIndex(InternalEngine.java:506)
org.elasticsearch.index.engine.internal.InternalEngine.index(InternalEngine.java:481)
org.elasticsearch.index.shard.service.InternalIndexShard.index(InternalIndexShard.java:404)
org.elasticsearch.action.index.TransportIndexAction.shardOperationOnReplica(TransportIndexAction.java:252)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$ReplicaOperationTransportHandler.messageReceived(TransportShardReplicationOperationAction.java:245)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$ReplicaOperationTransportHandler.messageReceived(TransportShardReplicationOperationAction.java:225)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:270)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
3.8% (376.4ms out of 10s) cpu usage by thread 'elasticsearch[dev4][index][T#3]'
8/10 snapshots sharing following 10 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:706)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.take(LinkedTransferQueue.java:1109)
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
2/10 snapshots sharing following 11 elements
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1529)
org.elasticsearch.index.engine.internal.InternalEngine.innerIndex(InternalEngine.java:553)
org.elasticsearch.index.engine.internal.InternalEngine.index(InternalEngine.java:481)
org.elasticsearch.index.shard.service.InternalIndexShard.index(InternalIndexShard.java:404)
org.elasticsearch.action.index.TransportIndexAction.shardOperationOnReplica(TransportIndexAction.java:252)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$ReplicaOperationTransportHandler.messageReceived(TransportShardReplicationOperationAction.java:245)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$ReplicaOperationTransportHandler.messageReceived(TransportShardReplicationOperationAction.java:225)
org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:270)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
3.6% (360.6ms out of 10s) cpu usage by thread 'elasticsearch[dev4][index][T#11]'
10/10 snapshots sharing following 10 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:706)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.take(LinkedTransferQueue.java:1109)
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
::: [dev9][k0qjuTuaSu24yWLqThrLdA][dev9.address][inet[/10.1.17.7:9300]]
5.8% (582.9ms out of 10s) cpu usage by thread 'elasticsearch[dev9][index][T#17]'
10/10 snapshots sharing following 10 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:706)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.take(LinkedTransferQueue.java:1109)
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
5.1% (514.3ms out of 10s) cpu usage by thread 'elasticsearch[dev9][index][T#21]'
10/10 snapshots sharing following 10 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:706)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.take(LinkedTransferQueue.java:1109)
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
5.1% (509.8ms out of 10s) cpu usage by thread 'elasticsearch[dev9][index][T#16]'
3/10 snapshots sharing following 28 elements
org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:85)
org.apache.lucene.index.TermsHash.flush(TermsHash.java:116)
org.apache.lucene.index.DocInverter.flush(DocInverter.java:53)
org.apache.lucene.index.DocFieldProcessor.flush(DocFieldProcessor.java:81)
org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:465)
org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:506)
org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:356)
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:438)
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1529)
org.elasticsearch.index.engine.internal.InternalEngine.innerIndex(InternalEngine.java:553)
org.elasticsearch.index.engine.internal.InternalEngine.index(InternalEngine.java:481)
org.elasticsearch.index.shard.service.InternalIndexShard.index(InternalIndexShard.java:404)
org.elasticsearch.action.index.TransportIndexAction.shardOperationOnPrimary(TransportIndexAction.java:210)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:556)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:433)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:330)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction.doExecute(TransportShardReplicationOperationAction.java:101)
org.elasticsearch.action.index.TransportIndexAction.innerExecute(TransportIndexAction.java:134)
org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:116)
org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:66)
org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:63)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:230)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:185)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:64)
org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction$AsyncSingleAction$1.run(TransportInstanceSingleOperationAction.java:192)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
2/10 snapshots sharing following 19 elements
org.elasticsearch.index.engine.internal.InternalEngine.innerIndex(InternalEngine.java:553)
org.elasticsearch.index.engine.internal.InternalEngine.index(InternalEngine.java:481)
org.elasticsearch.index.shard.service.InternalIndexShard.index(InternalIndexShard.java:404)
org.elasticsearch.action.index.TransportIndexAction.shardOperationOnPrimary(TransportIndexAction.java:210)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:556)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:433)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:330)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction.doExecute(TransportShardReplicationOperationAction.java:101)
org.elasticsearch.action.index.TransportIndexAction.innerExecute(TransportIndexAction.java:134)
org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:116)
org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:66)
org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:63)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:230)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:185)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:64)
org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction$AsyncSingleAction$1.run(TransportInstanceSingleOperationAction.java:192)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
5/10 snapshots sharing following 10 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:706)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.take(LinkedTransferQueue.java:1109)
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
4.9% (490.6ms out of 10s) cpu usage by thread 'elasticsearch[dev9][index][T#23]'
10/10 snapshots sharing following 10 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:706)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.take(LinkedTransferQueue.java:1109)
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
4.8% (480.4ms out of 10s) cpu usage by thread 'elasticsearch[dev9][index][T#12]'
10/10 snapshots sharing following 10 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:706)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.take(LinkedTransferQueue.java:1109)
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
4.8% (480.2ms out of 10s) cpu usage by thread 'elasticsearch[dev9][index][T#18]'
10/10 snapshots sharing following 10 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:706)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.take(LinkedTransferQueue.java:1109)
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
4.7% (473.9ms out of 10s) cpu usage by thread 'elasticsearch[dev9][index][T#15]'
9/10 snapshots sharing following 10 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:706)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.take(LinkedTransferQueue.java:1109)
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
unique snapshot
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:614)
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:209)
org.apache.lucene.index.DocInverterPerField.processFields(DocInverterPerField.java:165)
org.apache.lucene.index.DocFieldProcessor.processDocument(DocFieldProcessor.java:248)
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:253)
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:453)
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1529)
org.elasticsearch.index.engine.internal.InternalEngine.innerIndex(InternalEngine.java:553)
org.elasticsearch.index.engine.internal.InternalEngine.index(InternalEngine.java:481)
org.elasticsearch.index.shard.service.InternalIndexShard.index(InternalIndexShard.java:404)
org.elasticsearch.action.index.TransportIndexAction.shardOperationOnPrimary(TransportIndexAction.java:210)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:556)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:433)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:330)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction.doExecute(TransportShardReplicationOperationAction.java:101)
org.elasticsearch.action.index.TransportIndexAction.innerExecute(TransportIndexAction.java:134)
org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:116)
org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:66)
org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:63)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:230)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:185)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:64)
org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction$AsyncSingleAction$1.run(TransportInstanceSingleOperationAction.java:192)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
4.7% (472.9ms out of 10s) cpu usage by thread 'elasticsearch[dev9][index][T#22]'
6/10 snapshots sharing following 10 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:706)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.take(LinkedTransferQueue.java:1109)
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
2/10 snapshots sharing following 27 elements
org.apache.lucene.index.TermsHash.flush(TermsHash.java:116)
org.apache.lucene.index.DocInverter.flush(DocInverter.java:53)
org.apache.lucene.index.DocFieldProcessor.flush(DocFieldProcessor.java:81)
org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:465)
org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:506)
org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:356)
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:438)
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1529)
org.elasticsearch.index.engine.internal.InternalEngine.innerIndex(InternalEngine.java:553)
org.elasticsearch.index.engine.internal.InternalEngine.index(InternalEngine.java:481)
org.elasticsearch.index.shard.service.InternalIndexShard.index(InternalIndexShard.java:404)
org.elasticsearch.action.index.TransportIndexAction.shardOperationOnPrimary(TransportIndexAction.java:210)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:556)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:433)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:330)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction.doExecute(TransportShardReplicationOperationAction.java:101)
org.elasticsearch.action.index.TransportIndexAction.innerExecute(TransportIndexAction.java:134)
org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:116)
org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:66)
org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:63)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:230)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:185)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:64)
org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction$AsyncSingleAction$1.run(TransportInstanceSingleOperationAction.java:192)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
2/10 snapshots sharing following 19 elements
org.elasticsearch.index.engine.internal.InternalEngine.innerIndex(InternalEngine.java:553)
org.elasticsearch.index.engine.internal.InternalEngine.index(InternalEngine.java:481)
org.elasticsearch.index.shard.service.InternalIndexShard.index(InternalIndexShard.java:404)
org.elasticsearch.action.index.TransportIndexAction.shardOperationOnPrimary(TransportIndexAction.java:210)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:556)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:433)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:330)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction.doExecute(TransportShardReplicationOperationAction.java:101)
org.elasticsearch.action.index.TransportIndexAction.innerExecute(TransportIndexAction.java:134)
org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:116)
org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:66)
org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:63)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:230)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:185)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:64)
org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction$AsyncSingleAction$1.run(TransportInstanceSingleOperationAction.java:192)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
4.7% (472.5ms out of 10s) cpu usage by thread 'elasticsearch[dev9][index][T#5]'
7/10 snapshots sharing following 10 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:706)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.take(LinkedTransferQueue.java:1109)
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
3/10 snapshots sharing following 18 elements
org.elasticsearch.index.engine.internal.InternalEngine.index(InternalEngine.java:481)
org.elasticsearch.index.shard.service.InternalIndexShard.index(InternalIndexShard.java:404)
org.elasticsearch.action.index.TransportIndexAction.shardOperationOnPrimary(TransportIndexAction.java:210)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:556)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:433)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:330)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction.doExecute(TransportShardReplicationOperationAction.java:101)
org.elasticsearch.action.index.TransportIndexAction.innerExecute(TransportIndexAction.java:134)
org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:116)
org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:66)
org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:63)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:230)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:185)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:64)
org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction$AsyncSingleAction$1.run(TransportInstanceSingleOperationAction.java:192)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
4.7% (471.4ms out of 10s) cpu usage by thread 'elasticsearch[dev9][index][T#1]'
10/10 snapshots sharing following 10 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:706)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.take(LinkedTransferQueue.java:1109)
org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:744)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment