Skip to content

Instantly share code, notes, and snippets.

@eranid
Last active December 18, 2015 01:28
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save eranid/5703828 to your computer and use it in GitHub Desktop.
Save eranid/5703828 to your computer and use it in GitHub Desktop.
Several snapshots of "hot threads" and "/stats" endpoints
###################### 1 ################################
::: [Birely, Douglas][QINwIMqTROC5uAx5AatJ0A][inet[/*.*.*.*:*]]{aws_availability_zone=us-east-1d}
96.9% (484.5ms out of 500ms) cpu usage by thread 'elasticsearch[Birely, Douglas][cache][T#23]'
2/10 snapshots sharing following 4 elements
org.elasticsearch.index.cache.bloom.simple.SimpleBloomCache$BloomFilterLoader.run(SimpleBloomCache.java:199)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
2/10 snapshots sharing following 4 elements
org.elasticsearch.index.cache.bloom.simple.SimpleBloomCache$BloomFilterLoader.run(SimpleBloomCache.java:200)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
4/10 snapshots sharing following 10 elements
org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:162)
org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:270)
org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:136)
org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:111)
org.apache.lucene.index.TermBuffer.read(TermBuffer.java:82)
org.apache.lucene.index.SegmentTermEnum.next(SegmentTermEnum.java:133)
org.elasticsearch.index.cache.bloom.simple.SimpleBloomCache$BloomFilterLoader.run(SimpleBloomCache.java:203)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
2/10 snapshots sharing following 3 elements
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
96.4% (481.7ms out of 500ms) cpu usage by thread 'elasticsearch[Birely, Douglas][cache][T#26]'
4/10 snapshots sharing following 4 elements
org.elasticsearch.index.cache.bloom.simple.SimpleBloomCache$BloomFilterLoader.run(SimpleBloomCache.java:200)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
6/10 snapshots sharing following 3 elements
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
1.2% (6.1ms out of 500ms) cpu usage by thread 'elasticsearch[Birely, Douglas][refresh][T#1]'
4/10 snapshots sharing following 35 elements
sun.nio.ch.FileDispatcherImpl.pread0(Native Method)
sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:51)
sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:222)
sun.nio.ch.IOUtil.read(IOUtil.java:198)
sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:674)
org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:162)
org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:270)
org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:40)
org.apache.lucene.store.DataInput.readVInt(DataInput.java:107)
org.apache.lucene.store.BufferedIndexInput.readVInt(BufferedIndexInput.java:217)
org.apache.lucene.index.TermBuffer.read(TermBuffer.java:64)
org.apache.lucene.index.SegmentTermEnum.next(SegmentTermEnum.java:133)
org.apache.lucene.index.SegmentTermEnum.scanTo(SegmentTermEnum.java:174)
org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:236)
org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:172)
org.apache.lucene.index.SegmentTermDocs.seek(SegmentTermDocs.java:66)
org.apache.lucene.index.BufferedDeletesStream.applyTermDeletes(BufferedDeletesStream.java:371)
org.apache.lucene.index.BufferedDeletesStream.applyDeletes(BufferedDeletesStream.java:259)
org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3615)
org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3552)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:450)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:399)
org.apache.lucene.index.DirectoryReader.doOpenFromWriter(DirectoryReader.java:413)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:432)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:375)
org.apache.lucene.index.IndexReader.openIfChanged(IndexReader.java:508)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:109)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:57)
org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:137)
org.elasticsearch.index.engine.robin.RobinEngine.refresh(RobinEngine.java:769)
org.elasticsearch.index.shard.service.InternalIndexShard.refresh(InternalIndexShard.java:403)
org.elasticsearch.index.shard.service.InternalIndexShard$EngineRefresher$1.run(InternalIndexShard.java:733)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
6/10 snapshots sharing following 34 elements
sun.nio.ch.FileDispatcherImpl.pread0(Native Method)
sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:51)
sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:222)
sun.nio.ch.IOUtil.read(IOUtil.java:198)
sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:674)
org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:162)
org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:270)
org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:136)
org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:111)
org.apache.lucene.index.TermBuffer.read(TermBuffer.java:82)
org.apache.lucene.index.SegmentTermEnum.next(SegmentTermEnum.java:133)
org.apache.lucene.index.SegmentTermEnum.scanTo(SegmentTermEnum.java:174)
org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:236)
org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:172)
org.apache.lucene.index.SegmentTermDocs.seek(SegmentTermDocs.java:66)
org.apache.lucene.index.BufferedDeletesStream.applyTermDeletes(BufferedDeletesStream.java:371)
org.apache.lucene.index.BufferedDeletesStream.applyDeletes(BufferedDeletesStream.java:259)
org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3615)
org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3552)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:450)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:399)
org.apache.lucene.index.DirectoryReader.doOpenFromWriter(DirectoryReader.java:413)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:432)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:375)
org.apache.lucene.index.IndexReader.openIfChanged(IndexReader.java:508)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:109)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:57)
org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:137)
org.elasticsearch.index.engine.robin.RobinEngine.refresh(RobinEngine.java:769)
org.elasticsearch.index.shard.service.InternalIndexShard.refresh(InternalIndexShard.java:403)
org.elasticsearch.index.shard.service.InternalIndexShard$EngineRefresher$1.run(InternalIndexShard.java:733)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
###################### 2 ################################
::: [Birely, Douglas][QINwIMqTROC5uAx5AatJ0A][inet[/*.*.*.*:*]]{aws_availability_zone=us-east-1d}
97.9% (489.3ms out of 500ms) cpu usage by thread 'elasticsearch[Birely, Douglas][cache][T#23]'
8/10 snapshots sharing following 13 elements
sun.nio.ch.NativeThread.current(Native Method)
sun.nio.ch.NativeThreadSet.add(NativeThreadSet.java:46)
sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:670)
org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:162)
org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:270)
org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:136)
org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:111)
org.apache.lucene.index.TermBuffer.read(TermBuffer.java:82)
org.apache.lucene.index.SegmentTermEnum.next(SegmentTermEnum.java:133)
org.elasticsearch.index.cache.bloom.simple.SimpleBloomCache$BloomFilterLoader.run(SimpleBloomCache.java:203)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
2/10 snapshots sharing following 3 elements
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
0.2% (919micros out of 500ms) cpu usage by thread 'elasticsearch[Birely, Douglas][index][T#1666]'
10/10 snapshots sharing following 24 elements
sun.nio.ch.FileDispatcherImpl.pread0(Native Method)
sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:51)
sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:222)
sun.nio.ch.IOUtil.read(IOUtil.java:198)
sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:674)
org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:162)
org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:270)
org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:40)
org.apache.lucene.store.DataInput.readVInt(DataInput.java:107)
org.apache.lucene.store.BufferedIndexInput.readVInt(BufferedIndexInput.java:217)
org.apache.lucene.index.SegmentTermPositions.readDeltaPosition(SegmentTermPositions.java:76)
org.apache.lucene.index.SegmentTermPositions.nextPosition(SegmentTermPositions.java:72)
org.elasticsearch.common.lucene.uid.UidField.loadDocIdAndVersion(UidField.java:67)
org.elasticsearch.index.engine.robin.RobinEngine.get(RobinEngine.java:340)
org.elasticsearch.index.shard.service.InternalIndexShard.get(InternalIndexShard.java:393)
org.elasticsearch.index.get.ShardGetService.innerGet(ShardGetService.java:159)
org.elasticsearch.index.get.ShardGetService.get(ShardGetService.java:93)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:219)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:211)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:85)
org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction$AsyncSingleAction$1.run(TransportInstanceSingleOperationAction.java:191)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
0.1% (594.4micros out of 500ms) cpu usage by thread 'elasticsearch[Birely, Douglas][scheduler][T#1]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
###################### 3 ################################
::: [Birely, Douglas][QINwIMqTROC5uAx5AatJ0A][inet[/*.*.*.*:*]]{aws_availability_zone=us-east-1d}
98.4% (492.2ms out of 500ms) cpu usage by thread 'elasticsearch[Birely, Douglas][cache][T#27]'
7/10 snapshots sharing following 13 elements
sun.nio.ch.NativeThread.current(Native Method)
sun.nio.ch.NativeThreadSet.add(NativeThreadSet.java:46)
sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:670)
org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:162)
org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:270)
org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:136)
org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:111)
org.apache.lucene.index.TermBuffer.read(TermBuffer.java:82)
org.apache.lucene.index.SegmentTermEnum.next(SegmentTermEnum.java:133)
org.elasticsearch.index.cache.bloom.simple.SimpleBloomCache$BloomFilterLoader.run(SimpleBloomCache.java:203)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
3/10 snapshots sharing following 3 elements
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
2.5% (12.3ms out of 500ms) cpu usage by thread 'elasticsearch[Birely, Douglas][search][T#681]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
2.2% (11.2ms out of 500ms) cpu usage by thread 'elasticsearch[Birely, Douglas][refresh][T#1]'
10/10 snapshots sharing following 35 elements
sun.nio.ch.FileDispatcherImpl.pread0(Native Method)
sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:51)
sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:222)
sun.nio.ch.IOUtil.read(IOUtil.java:198)
sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:674)
org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:162)
org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:270)
org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:40)
org.apache.lucene.store.DataInput.readVInt(DataInput.java:107)
org.apache.lucene.store.BufferedIndexInput.readVInt(BufferedIndexInput.java:217)
org.apache.lucene.index.TermBuffer.read(TermBuffer.java:64)
org.apache.lucene.index.SegmentTermEnum.next(SegmentTermEnum.java:133)
org.apache.lucene.index.SegmentTermEnum.scanTo(SegmentTermEnum.java:174)
org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:236)
org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:172)
org.apache.lucene.index.SegmentTermDocs.seek(SegmentTermDocs.java:66)
org.apache.lucene.index.BufferedDeletesStream.applyTermDeletes(BufferedDeletesStream.java:371)
org.apache.lucene.index.BufferedDeletesStream.applyDeletes(BufferedDeletesStream.java:259)
org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3615)
org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3552)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:450)
org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:399)
org.apache.lucene.index.DirectoryReader.doOpenFromWriter(DirectoryReader.java:413)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:432)
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:375)
org.apache.lucene.index.IndexReader.openIfChanged(IndexReader.java:508)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:109)
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:57)
org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:137)
org.elasticsearch.index.engine.robin.RobinEngine.refresh(RobinEngine.java:769)
org.elasticsearch.index.shard.service.InternalIndexShard.refresh(InternalIndexShard.java:403)
org.elasticsearch.index.shard.service.InternalIndexShard$EngineRefresher$1.run(InternalIndexShard.java:733)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
###################### 4 ################################
::: [Birely, Douglas][QINwIMqTROC5uAx5AatJ0A][inet[/*.*.*.*:*]]{aws_availability_zone=us-east-1d}
84.0% (419.7ms out of 500ms) cpu usage by thread 'elasticsearch[Birely, Douglas][cache][T#27]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
1.4% (7.1ms out of 500ms) cpu usage by thread 'elasticsearch[Birely, Douglas][index][T#1666]'
3/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
7/10 snapshots sharing following 29 elements
sun.nio.ch.FileDispatcherImpl.pread0(Native Method)
sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:51)
sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:222)
sun.nio.ch.IOUtil.read(IOUtil.java:198)
sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:674)
org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:162)
org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:270)
org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:40)
org.apache.lucene.store.DataInput.readVInt(DataInput.java:107)
org.apache.lucene.store.BufferedIndexInput.readVInt(BufferedIndexInput.java:217)
org.apache.lucene.index.TermBuffer.read(TermBuffer.java:64)
org.apache.lucene.index.SegmentTermEnum.next(SegmentTermEnum.java:133)
org.apache.lucene.index.SegmentTermEnum.scanTo(SegmentTermEnum.java:174)
org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:236)
org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:172)
org.apache.lucene.index.SegmentTermDocs.seek(SegmentTermDocs.java:66)
org.apache.lucene.index.IndexReader.termPositions(IndexReader.java:1340)
org.elasticsearch.common.lucene.uid.UidField.loadDocIdAndVersion(UidField.java:59)
org.elasticsearch.index.engine.robin.RobinEngine.get(RobinEngine.java:340)
org.elasticsearch.index.shard.service.InternalIndexShard.get(InternalIndexShard.java:393)
org.elasticsearch.index.get.ShardGetService.innerGet(ShardGetService.java:159)
org.elasticsearch.index.get.ShardGetService.get(ShardGetService.java:93)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:219)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:211)
org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:85)
org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction$AsyncSingleAction$1.run(TransportInstanceSingleOperationAction.java:191)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
0.4% (2ms out of 500ms) cpu usage by thread 'elasticsearch[Birely, Douglas][index][T#1653]'
10/10 snapshots sharing following 9 elements
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
java.lang.Thread.run(Thread.java:722)
{
"ok" : true,
"_shards" : {
"total" : 8,
"successful" : 4,
"failed" : 0
},
"_all" : {
"primaries" : {
"docs" : {
"count" : 148605093,
"deleted" : 7201672
},
"store" : {
"size" : "265.5gb",
"size_in_bytes" : 285082233386,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"indexing" : {
"index_total" : 1871148,
"index_time" : "1.5d",
"index_time_in_millis" : 129613503,
"index_current" : 0,
"delete_total" : 0,
"delete_time" : "0s",
"delete_time_in_millis" : 0,
"delete_current" : 0
},
"get" : {
"total" : 2140184,
"time" : "3.2d",
"time_in_millis" : 278610855,
"exists_total" : 2140184,
"exists_time" : "3.2d",
"exists_time_in_millis" : 278610855,
"missing_total" : 0,
"missing_time" : "0s",
"missing_time_in_millis" : 0,
"current" : 1
},
"search" : {
"query_total" : 849986,
"query_time" : "12.7h",
"query_time_in_millis" : 45729737,
"query_current" : 0,
"fetch_total" : 849983,
"fetch_time" : "13.1h",
"fetch_time_in_millis" : 47261104,
"fetch_current" : 0
},
"refresh" : {
"total" : 206306,
"total_time" : "14.2h",
"total_time_in_millis" : 51338800
},
"warmer" : {
"current" : 0,
"total" : 206637,
"total_time" : "17.1m",
"total_time_in_millis" : 1026731
}
},
"total" : {
"docs" : {
"count" : 148605093,
"deleted" : 7201672
},
"store" : {
"size" : "265.5gb",
"size_in_bytes" : 285082233386,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"indexing" : {
"index_total" : 1871148,
"index_time" : "1.5d",
"index_time_in_millis" : 129613503,
"index_current" : 0,
"delete_total" : 0,
"delete_time" : "0s",
"delete_time_in_millis" : 0,
"delete_current" : 0
},
"get" : {
"total" : 2140184,
"time" : "3.2d",
"time_in_millis" : 278610855,
"exists_total" : 2140184,
"exists_time" : "3.2d",
"exists_time_in_millis" : 278610855,
"missing_total" : 0,
"missing_time" : "0s",
"missing_time_in_millis" : 0,
"current" : 1
},
"search" : {
"query_total" : 849986,
"query_time" : "12.7h",
"query_time_in_millis" : 45729737,
"query_current" : 0,
"fetch_total" : 849983,
"fetch_time" : "13.1h",
"fetch_time_in_millis" : 47261104,
"fetch_current" : 0
},
"refresh" : {
"total" : 206306,
"total_time" : "14.2h",
"total_time_in_millis" : 51338800
},
"warmer" : {
"current" : 0,
"total" : 206637,
"total_time" : "17.1m",
"total_time_in_millis" : 1026731
}
},
"indices" : {
"my_index" : {
"primaries" : {
"docs" : {
"count" : 2138844,
"deleted" : 17
},
"store" : {
"size" : "1.8gb",
"size_in_bytes" : 1986630858,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"indexing" : {
"index_total" : 0,
"index_time" : "0s",
"index_time_in_millis" : 0,
"index_current" : 0,
"delete_total" : 0,
"delete_time" : "0s",
"delete_time_in_millis" : 0,
"delete_current" : 0
},
"get" : {
"total" : 0,
"time" : "0s",
"time_in_millis" : 0,
"exists_total" : 0,
"exists_time" : "0s",
"exists_time_in_millis" : 0,
"missing_total" : 0,
"missing_time" : "0s",
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"query_total" : 1,
"query_time" : "185ms",
"query_time_in_millis" : 185,
"query_current" : 0,
"fetch_total" : 1,
"fetch_time" : "282ms",
"fetch_time_in_millis" : 282,
"fetch_current" : 0
},
"refresh" : {
"total" : 1,
"total_time" : "0s",
"total_time_in_millis" : 0
},
"warmer" : {
"current" : 0,
"total" : 2,
"total_time" : "1ms",
"total_time_in_millis" : 1
}
},
"total" : {
"docs" : {
"count" : 2138844,
"deleted" : 17
},
"store" : {
"size" : "1.8gb",
"size_in_bytes" : 1986630858,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"indexing" : {
"index_total" : 0,
"index_time" : "0s",
"index_time_in_millis" : 0,
"index_current" : 0,
"delete_total" : 0,
"delete_time" : "0s",
"delete_time_in_millis" : 0,
"delete_current" : 0
},
"get" : {
"total" : 0,
"time" : "0s",
"time_in_millis" : 0,
"exists_total" : 0,
"exists_time" : "0s",
"exists_time_in_millis" : 0,
"missing_total" : 0,
"missing_time" : "0s",
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"query_total" : 1,
"query_time" : "185ms",
"query_time_in_millis" : 185,
"query_current" : 0,
"fetch_total" : 1,
"fetch_time" : "282ms",
"fetch_time_in_millis" : 282,
"fetch_current" : 0
},
"refresh" : {
"total" : 1,
"total_time" : "0s",
"total_time_in_millis" : 0
},
"warmer" : {
"current" : 0,
"total" : 2,
"total_time" : "1ms",
"total_time_in_millis" : 1
}
}
},
"my_index_logins" : {
"primaries" : {
"docs" : {
"count" : 71011,
"deleted" : 152
},
"store" : {
"size" : "33.3mb",
"size_in_bytes" : 35020944,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"indexing" : {
"index_total" : 0,
"index_time" : "0s",
"index_time_in_millis" : 0,
"index_current" : 0,
"delete_total" : 0,
"delete_time" : "0s",
"delete_time_in_millis" : 0,
"delete_current" : 0
},
"get" : {
"total" : 0,
"time" : "0s",
"time_in_millis" : 0,
"exists_total" : 0,
"exists_time" : "0s",
"exists_time_in_millis" : 0,
"missing_total" : 0,
"missing_time" : "0s",
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"query_total" : 1,
"query_time" : "80ms",
"query_time_in_millis" : 80,
"query_current" : 0,
"fetch_total" : 0,
"fetch_time" : "0s",
"fetch_time_in_millis" : 0,
"fetch_current" : 0
},
"refresh" : {
"total" : 1,
"total_time" : "0s",
"total_time_in_millis" : 0
},
"warmer" : {
"current" : 0,
"total" : 2,
"total_time" : "0s",
"total_time_in_millis" : 0
}
},
"total" : {
"docs" : {
"count" : 71011,
"deleted" : 152
},
"store" : {
"size" : "33.3mb",
"size_in_bytes" : 35020944,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"indexing" : {
"index_total" : 0,
"index_time" : "0s",
"index_time_in_millis" : 0,
"index_current" : 0,
"delete_total" : 0,
"delete_time" : "0s",
"delete_time_in_millis" : 0,
"delete_current" : 0
},
"get" : {
"total" : 0,
"time" : "0s",
"time_in_millis" : 0,
"exists_total" : 0,
"exists_time" : "0s",
"exists_time_in_millis" : 0,
"missing_total" : 0,
"missing_time" : "0s",
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"query_total" : 1,
"query_time" : "80ms",
"query_time_in_millis" : 80,
"query_current" : 0,
"fetch_total" : 0,
"fetch_time" : "0s",
"fetch_time_in_millis" : 0,
"fetch_current" : 0
},
"refresh" : {
"total" : 1,
"total_time" : "0s",
"total_time_in_millis" : 0
},
"warmer" : {
"current" : 0,
"total" : 2,
"total_time" : "0s",
"total_time_in_millis" : 0
}
}
},
"test_my_index" : {
"primaries" : {
"docs" : {
"count" : 3622,
"deleted" : 14
},
"store" : {
"size" : "2.4mb",
"size_in_bytes" : 2541612,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"indexing" : {
"index_total" : 0,
"index_time" : "0s",
"index_time_in_millis" : 0,
"index_current" : 0,
"delete_total" : 0,
"delete_time" : "0s",
"delete_time_in_millis" : 0,
"delete_current" : 0
},
"get" : {
"total" : 0,
"time" : "0s",
"time_in_millis" : 0,
"exists_total" : 0,
"exists_time" : "0s",
"exists_time_in_millis" : 0,
"missing_total" : 0,
"missing_time" : "0s",
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"query_total" : 1,
"query_time" : "127ms",
"query_time_in_millis" : 127,
"query_current" : 0,
"fetch_total" : 0,
"fetch_time" : "0s",
"fetch_time_in_millis" : 0,
"fetch_current" : 0
},
"refresh" : {
"total" : 1,
"total_time" : "0s",
"total_time_in_millis" : 0
},
"warmer" : {
"current" : 0,
"total" : 2,
"total_time" : "0s",
"total_time_in_millis" : 0
}
},
"total" : {
"docs" : {
"count" : 3622,
"deleted" : 14
},
"store" : {
"size" : "2.4mb",
"size_in_bytes" : 2541612,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"indexing" : {
"index_total" : 0,
"index_time" : "0s",
"index_time_in_millis" : 0,
"index_current" : 0,
"delete_total" : 0,
"delete_time" : "0s",
"delete_time_in_millis" : 0,
"delete_current" : 0
},
"get" : {
"total" : 0,
"time" : "0s",
"time_in_millis" : 0,
"exists_total" : 0,
"exists_time" : "0s",
"exists_time_in_millis" : 0,
"missing_total" : 0,
"missing_time" : "0s",
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"query_total" : 1,
"query_time" : "127ms",
"query_time_in_millis" : 127,
"query_current" : 0,
"fetch_total" : 0,
"fetch_time" : "0s",
"fetch_time_in_millis" : 0,
"fetch_current" : 0
},
"refresh" : {
"total" : 1,
"total_time" : "0s",
"total_time_in_millis" : 0
},
"warmer" : {
"current" : 0,
"total" : 2,
"total_time" : "0s",
"total_time_in_millis" : 0
}
}
},
"my_index_more" : {
"primaries" : {
"docs" : {
"count" : 146391616,
"deleted" : 7201489
},
"store" : {
"size" : "263.6gb",
"size_in_bytes" : 283058039972,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"indexing" : {
"index_total" : 1871148,
"index_time" : "1.5d",
"index_time_in_millis" : 129613503,
"index_current" : 0,
"delete_total" : 0,
"delete_time" : "0s",
"delete_time_in_millis" : 0,
"delete_current" : 0
},
"get" : {
"total" : 2140184,
"time" : "3.2d",
"time_in_millis" : 278610855,
"exists_total" : 2140184,
"exists_time" : "3.2d",
"exists_time_in_millis" : 278610855,
"missing_total" : 0,
"missing_time" : "0s",
"missing_time_in_millis" : 0,
"current" : 1
},
"search" : {
"query_total" : 849983,
"query_time" : "12.7h",
"query_time_in_millis" : 45729345,
"query_current" : 0,
"fetch_total" : 849982,
"fetch_time" : "13.1h",
"fetch_time_in_millis" : 47260822,
"fetch_current" : 0
},
"refresh" : {
"total" : 206303,
"total_time" : "14.2h",
"total_time_in_millis" : 51338800
},
"warmer" : {
"current" : 0,
"total" : 206631,
"total_time" : "17.1m",
"total_time_in_millis" : 1026730
}
},
"total" : {
"docs" : {
"count" : 146391616,
"deleted" : 7201489
},
"store" : {
"size" : "263.6gb",
"size_in_bytes" : 283058039972,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"indexing" : {
"index_total" : 1871148,
"index_time" : "1.5d",
"index_time_in_millis" : 129613503,
"index_current" : 0,
"delete_total" : 0,
"delete_time" : "0s",
"delete_time_in_millis" : 0,
"delete_current" : 0
},
"get" : {
"total" : 2140184,
"time" : "3.2d",
"time_in_millis" : 278610855,
"exists_total" : 2140184,
"exists_time" : "3.2d",
"exists_time_in_millis" : 278610855,
"missing_total" : 0,
"missing_time" : "0s",
"missing_time_in_millis" : 0,
"current" : 1
},
"search" : {
"query_total" : 849983,
"query_time" : "12.7h",
"query_time_in_millis" : 45729345,
"query_current" : 0,
"fetch_total" : 849982,
"fetch_time" : "13.1h",
"fetch_time_in_millis" : 47260822,
"fetch_current" : 0
},
"refresh" : {
"total" : 206303,
"total_time" : "14.2h",
"total_time_in_millis" : 51338800
},
"warmer" : {
"current" : 0,
"total" : 206631,
"total_time" : "17.1m",
"total_time_in_millis" : 1026730
}
}
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment