Skip to content

Instantly share code, notes, and snippets.

/1.java Secret

Created October 2, 2012 08:14
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save anonymous/f2826104eb5b288d3fb0 to your computer and use it in GitHub Desktop.
Save anonymous/f2826104eb5b288d3fb0 to your computer and use it in GitHub Desktop.
Download ZIP
ES 100% processor usage
Raw
1.java
::: [es1][IiY2mhoZRFWN1ojbL-5Ejw][inet[/10.29.212.95:9300]]{rack_id=es1_rack}
100.3% (501.3ms out of 500ms) cpu usage by thread 'elasticsearch[es1][index][T#3]'
4/10 snapshots sharing following 27 elements
org.apache.lucene.search.FilteredQuery$2.advance(FilteredQuery.java:201)
org.elasticsearch.index.search.nested.IncludeAllChildrenQuery$IncludeAllChildrenScorer.advance(IncludeAllChildrenQuery.java:195)
org.apache.lucene.search.FilteredQuery$2.advanceToNextCommonDoc(FilteredQuery.java:181)
org.apache.lucene.search.FilteredQuery$2.nextDoc(FilteredQuery.java:193)
org.apache.lucene.index.BufferedDeletesStream.applyQueryDeletes(BufferedDeletesStream.java:410)
org.apache.lucene.index.BufferedDeletesStream.applyDeletes(BufferedDeletesStream.java:260)
org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3615)
org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3552)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:450)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:399)
org.apache.lucene.index.DirectoryReader.doOpenFromWriter(DirectoryReader.java:413)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:432)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:375)
org.apache.lucene.index.IndexReader.openIfChanged(IndexReader.java:508)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:109)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:57)
org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:137)
org.elasticsearch.index.engine.robin.RobinEngine.refresh(RobinEngine.java:767)
org.elasticsearch.index.engine.robin.RobinEngine.refreshVersioningTable(RobinEngine.java:926)
org.elasticsearch.index.engine.robin.RobinEngine.delete(RobinEngine.java:727)
org.elasticsearch.index.shard.service.InternalIndexShard.deleteByQuery(InternalIndexShard.java:381)
org.elasticsearch.action.deletebyquery.TransportShardDeleteByQueryAction.shardOperationOnPrimary(TransportShardDeleteByQueryAction.java:96)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:532)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$1.run(TransportShardReplicationOperationAction.java:430)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
6/10 snapshots sharing following 25 elements
org.apache.lucene.search.FilteredQuery$2.advanceToNextCommonDoc(FilteredQuery.java:181)
org.apache.lucene.search.FilteredQuery$2.nextDoc(FilteredQuery.java:193)
org.apache.lucene.index.BufferedDeletesStream.applyQueryDeletes(BufferedDeletesStream.java:410)
org.apache.lucene.index.BufferedDeletesStream.applyDeletes(BufferedDeletesStream.java:260)
org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3615)
org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3552)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:450)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:399)
org.apache.lucene.index.DirectoryReader.doOpenFromWriter(DirectoryReader.java:413)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:432)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:375)
org.apache.lucene.index.IndexReader.openIfChanged(IndexReader.java:508)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:109)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:57)
org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:137)
org.elasticsearch.index.engine.robin.RobinEngine.refresh(RobinEngine.java:767)
org.elasticsearch.index.engine.robin.RobinEngine.refreshVersioningTable(RobinEngine.java:926)
org.elasticsearch.index.engine.robin.RobinEngine.delete(RobinEngine.java:727)
org.elasticsearch.index.shard.service.InternalIndexShard.deleteByQuery(InternalIndexShard.java:381)
org.elasticsearch.action.deletebyquery.TransportShardDeleteByQueryAction.shardOperationOnPrimary(TransportShardDeleteByQueryAction.java:96)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:532)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$1.run(TransportShardReplicationOperationAction.java:430)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
0.1% (670.1micros out of 500ms) cpu usage by thread 'elasticsearch[es1][scheduler][T#1]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
0.1% (517.8micros out of 500ms) cpu usage by thread 'elasticsearch[es1][transport_server_worker][T#4]{New I/O worker #12}'
10/10 snapshots sharing following 13 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:223)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
::: [es2][A82gG1kgTZeD3Gdmt5vt2Q][inet[/10.140.16.186:9300]]{rack_id=es2_rack}
0.1% (315.3micros out of 500ms) cpu usage by thread 'elasticsearch[es2][transport_client_worker][T#4]{New I/O worker #4}'
10/10 snapshots sharing following 13 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:223)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
0.1% (307.3micros out of 500ms) cpu usage by thread 'elasticsearch[es2][transport_client_worker][T#8]{New I/O worker #8}'
10/10 snapshots sharing following 13 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:223)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
0.1% (305.1micros out of 500ms) cpu usage by thread 'elasticsearch[es2][http_server_worker][T#2]{New I/O worker #18}'
10/10 snapshots sharing following 13 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:223)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
Raw
2.java
::: [es1][IiY2mhoZRFWN1ojbL-5Ejw][inet[/10.29.212.95:9300]]{rack_id=es1_rack}
100.3% (501.3ms out of 500ms) cpu usage by thread 'elasticsearch[es1][index][T#3]'
10/10 snapshots sharing following 25 elements
org.apache.lucene.search.FilteredQuery$2.advanceToNextCommonDoc(FilteredQuery.java:181)
org.apache.lucene.search.FilteredQuery$2.nextDoc(FilteredQuery.java:193)
org.apache.lucene.index.BufferedDeletesStream.applyQueryDeletes(BufferedDeletesStream.java:410)
org.apache.lucene.index.BufferedDeletesStream.applyDeletes(BufferedDeletesStream.java:260)
org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3615)
org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3552)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:450)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:399)
org.apache.lucene.index.DirectoryReader.doOpenFromWriter(DirectoryReader.java:413)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:432)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:375)
org.apache.lucene.index.IndexReader.openIfChanged(IndexReader.java:508)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:109)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:57)
org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:137)
org.elasticsearch.index.engine.robin.RobinEngine.refresh(RobinEngine.java:767)
org.elasticsearch.index.engine.robin.RobinEngine.refreshVersioningTable(RobinEngine.java:926)
org.elasticsearch.index.engine.robin.RobinEngine.delete(RobinEngine.java:727)
org.elasticsearch.index.shard.service.InternalIndexShard.deleteByQuery(InternalIndexShard.java:381)
org.elasticsearch.action.deletebyquery.TransportShardDeleteByQueryAction.shardOperationOnPrimary(TransportShardDeleteByQueryAction.java:96)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:532)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$1.run(TransportShardReplicationOperationAction.java:430)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
0.2% (947.1micros out of 500ms) cpu usage by thread 'elasticsearch[es1][transport_client_worker][T#6]{New I/O worker #6}'
10/10 snapshots sharing following 13 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:223)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
0.2% (939micros out of 500ms) cpu usage by thread 'elasticsearch[es1][scheduler][T#1]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
::: [es2][A82gG1kgTZeD3Gdmt5vt2Q][inet[/10.140.16.186:9300]]{rack_id=es2_rack}
0.2% (788.6micros out of 500ms) cpu usage by thread 'elasticsearch[es2][transport_server_worker][T#2]{New I/O worker #10}'
10/10 snapshots sharing following 13 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:223)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
0.1% (572.5micros out of 500ms) cpu usage by thread 'elasticsearch[es2][scheduler][T#1]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
0.1% (370.1micros out of 500ms) cpu usage by thread 'elasticsearch[es2][transport_server_worker][T#7]{New I/O worker #15}'
10/10 snapshots sharing following 13 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:223)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
Raw
3.java
::: [es1][IiY2mhoZRFWN1ojbL-5Ejw][inet[/10.29.212.95:9300]]{rack_id=es1_rack}
100.3% (501.4ms out of 500ms) cpu usage by thread 'elasticsearch[es1][index][T#3]'
10/10 snapshots sharing following 25 elements
org.apache.lucene.search.FilteredQuery$2.advanceToNextCommonDoc(FilteredQuery.java:181)
org.apache.lucene.search.FilteredQuery$2.nextDoc(FilteredQuery.java:193)
org.apache.lucene.index.BufferedDeletesStream.applyQueryDeletes(BufferedDeletesStream.java:410)
org.apache.lucene.index.BufferedDeletesStream.applyDeletes(BufferedDeletesStream.java:260)
org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3615)
org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3552)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:450)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:399)
org.apache.lucene.index.DirectoryReader.doOpenFromWriter(DirectoryReader.java:413)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:432)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:375)
org.apache.lucene.index.IndexReader.openIfChanged(IndexReader.java:508)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:109)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:57)
org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:137)
org.elasticsearch.index.engine.robin.RobinEngine.refresh(RobinEngine.java:767)
org.elasticsearch.index.engine.robin.RobinEngine.refreshVersioningTable(RobinEngine.java:926)
org.elasticsearch.index.engine.robin.RobinEngine.delete(RobinEngine.java:727)
org.elasticsearch.index.shard.service.InternalIndexShard.deleteByQuery(InternalIndexShard.java:381)
org.elasticsearch.action.deletebyquery.TransportShardDeleteByQueryAction.shardOperationOnPrimary(TransportShardDeleteByQueryAction.java:96)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:532)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$1.run(TransportShardReplicationOperationAction.java:430)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
0.2% (1.1ms out of 500ms) cpu usage by thread 'elasticsearch[es1][transport_client_worker][T#6]{New I/O worker #6}'
10/10 snapshots sharing following 13 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:223)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
0.2% (1ms out of 500ms) cpu usage by thread 'elasticsearch[es1][scheduler][T#1]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
::: [es2][A82gG1kgTZeD3Gdmt5vt2Q][inet[/10.140.16.186:9300]]{rack_id=es2_rack}
0.2% (761.9micros out of 500ms) cpu usage by thread 'elasticsearch[es2][transport_server_worker][T#2]{New I/O worker #10}'
10/10 snapshots sharing following 13 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:223)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
0.1% (402.4micros out of 500ms) cpu usage by thread 'elasticsearch[es2][http_server_worker][T#7]{New I/O worker #23}'
10/10 snapshots sharing following 13 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:223)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
0.1% (387micros out of 500ms) cpu usage by thread 'elasticsearch[es2][transport_client_worker][T#1]{New I/O worker #2}'
10/10 snapshots sharing following 13 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:223)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
Raw
4.java
::: [es1][IiY2mhoZRFWN1ojbL-5Ejw][inet[/10.29.212.95:9300]]{rack_id=es1_rack}
100.2% (501.1ms out of 500ms) cpu usage by thread 'elasticsearch[es1][index][T#3]'
7/10 snapshots sharing following 27 elements
org.apache.lucene.search.FilteredQuery$2.advance(FilteredQuery.java:201)
org.elasticsearch.index.search.nested.IncludeAllChildrenQuery$IncludeAllChildrenScorer.advance(IncludeAllChildrenQuery.java:195)
org.apache.lucene.search.FilteredQuery$2.advanceToNextCommonDoc(FilteredQuery.java:181)
org.apache.lucene.search.FilteredQuery$2.nextDoc(FilteredQuery.java:193)
org.apache.lucene.index.BufferedDeletesStream.applyQueryDeletes(BufferedDeletesStream.java:410)
org.apache.lucene.index.BufferedDeletesStream.applyDeletes(BufferedDeletesStream.java:260)
org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3615)
org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3552)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:450)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:399)
org.apache.lucene.index.DirectoryReader.doOpenFromWriter(DirectoryReader.java:413)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:432)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:375)
org.apache.lucene.index.IndexReader.openIfChanged(IndexReader.java:508)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:109)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:57)
org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:137)
org.elasticsearch.index.engine.robin.RobinEngine.refresh(RobinEngine.java:767)
org.elasticsearch.index.engine.robin.RobinEngine.refreshVersioningTable(RobinEngine.java:926)
org.elasticsearch.index.engine.robin.RobinEngine.delete(RobinEngine.java:727)
org.elasticsearch.index.shard.service.InternalIndexShard.deleteByQuery(InternalIndexShard.java:381)
org.elasticsearch.action.deletebyquery.TransportShardDeleteByQueryAction.shardOperationOnPrimary(TransportShardDeleteByQueryAction.java:96)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:532)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$1.run(TransportShardReplicationOperationAction.java:430)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
3/10 snapshots sharing following 25 elements
org.apache.lucene.search.FilteredQuery$2.advanceToNextCommonDoc(FilteredQuery.java:181)
org.apache.lucene.search.FilteredQuery$2.nextDoc(FilteredQuery.java:193)
org.apache.lucene.index.BufferedDeletesStream.applyQueryDeletes(BufferedDeletesStream.java:410)
org.apache.lucene.index.BufferedDeletesStream.applyDeletes(BufferedDeletesStream.java:260)
org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3615)
org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3552)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:450)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:399)
org.apache.lucene.index.DirectoryReader.doOpenFromWriter(DirectoryReader.java:413)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:432)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:375)
org.apache.lucene.index.IndexReader.openIfChanged(IndexReader.java:508)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:109)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:57)
org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:137)
org.elasticsearch.index.engine.robin.RobinEngine.refresh(RobinEngine.java:767)
org.elasticsearch.index.engine.robin.RobinEngine.refreshVersioningTable(RobinEngine.java:926)
org.elasticsearch.index.engine.robin.RobinEngine.delete(RobinEngine.java:727)
org.elasticsearch.index.shard.service.InternalIndexShard.deleteByQuery(InternalIndexShard.java:381)
org.elasticsearch.action.deletebyquery.TransportShardDeleteByQueryAction.shardOperationOnPrimary(TransportShardDeleteByQueryAction.java:96)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:532)
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$1.run(TransportShardReplicationOperationAction.java:430)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
0.3% (1.4ms out of 500ms) cpu usage by thread 'elasticsearch[es1][scheduler][T#1]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
0.2% (948.8micros out of 500ms) cpu usage by thread 'elasticsearch[es1][transport_client_worker][T#6]{New I/O worker #6}'
10/10 snapshots sharing following 13 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:223)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
::: [es2][A82gG1kgTZeD3Gdmt5vt2Q][inet[/10.140.16.186:9300]]{rack_id=es2_rack}
0.2% (795.8micros out of 500ms) cpu usage by thread 'elasticsearch[es2][transport_server_worker][T#2]{New I/O worker #10}'
10/10 snapshots sharing following 13 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:223)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
0.1% (690.1micros out of 500ms) cpu usage by thread 'elasticsearch[es2][scheduler][T#1]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
0.1% (406micros out of 500ms) cpu usage by thread 'elasticsearch[es2][transport_client_worker][T#1]{New I/O worker #2}'
10/10 snapshots sharing following 13 elements
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:223)
org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
java.lang.Thread.run(Thread.java:722)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment