Created
December 12, 2019 23:55
-
-
Save vinicioflores/5a150b1a25e094ef16a15ed9690b4b51 to your computer and use it in GitHub Desktop.
Hot Threads ES
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ::: {arm-or-001_master}{vACvBzd5RiqvDDgWNvY6EQ}{TrPh5fRwRjKvxpYgV2h_oA}{plxcq8197.myserver.com}{x.x.x.x:9301}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.447Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 0.0% (50.8micros out of 500ms) cpu usage by thread 'elasticsearch[arm-or-001_master][[timer]]' | |
| 10/10 snapshots sharing following 2 elements | |
| java.lang.Thread.sleep(Native Method) | |
| org.elasticsearch.threadpool.ThreadPool$CachedTimeThread.run(ThreadPool.java:541) | |
| ::: {arm-or-009_data}{f0aSRTkDQommICaE9nQRBg}{JMnusso4RfCBpFObfysMTA}{plxcq8205.myserver.com}{x.x.x.x:9300}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.410Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 66.4% (331.7ms out of 500ms) cpu usage by thread 'elasticsearch[arm-or-009_data][management][T#2]' | |
| 4/10 snapshots sharing following 37 elements | |
| java.io.UnixFileSystem.canonicalize0(Native Method) | |
| java.io.UnixFileSystem.canonicalize(UnixFileSystem.java:172) | |
| java.io.File.getCanonicalPath(File.java:618) | |
| java.io.FilePermission$1.run(FilePermission.java:215) | |
| java.io.FilePermission$1.run(FilePermission.java:203) | |
| java.security.AccessController.doPrivileged(Native Method) | |
| java.io.FilePermission.init(FilePermission.java:203) | |
| java.io.FilePermission.<init>(FilePermission.java:277) | |
| java.lang.SecurityManager.checkRead(SecurityManager.java:888) | |
| sun.nio.fs.UnixPath.checkRead(UnixPath.java:795) | |
| sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49) | |
| sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) | |
| sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) | |
| java.nio.file.Files.readAttributes(Files.java:1737) | |
| java.nio.file.Files.size(Files.java:2332) | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 3/10 snapshots sharing following 26 elements | |
| sun.nio.fs.UnixNativeDispatcher.readdir(Native Method) | |
| sun.nio.fs.UnixDirectoryStream$UnixDirectoryIterator.readNextEntry(UnixDirectoryStream.java:168) | |
| sun.nio.fs.UnixDirectoryStream$UnixDirectoryIterator.hasNext(UnixDirectoryStream.java:201) | |
| org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:216) | |
| org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:234) | |
| org.apache.lucene.store.FilterDirectory.listAll(FilterDirectory.java:57) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1445) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 2/10 snapshots sharing following 29 elements | |
| java.lang.SecurityManager.checkRead(SecurityManager.java:888) | |
| sun.nio.fs.UnixPath.checkRead(UnixPath.java:795) | |
| sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49) | |
| sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) | |
| sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) | |
| java.nio.file.Files.readAttributes(Files.java:1737) | |
| java.nio.file.Files.size(Files.java:2332) | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| unique snapshot | |
| sun.nio.fs.UnixNativeDispatcher.stat0(Native Method) | |
| sun.nio.fs.UnixNativeDispatcher.stat(UnixNativeDispatcher.java:286) | |
| sun.nio.fs.UnixFileAttributes.get(UnixFileAttributes.java:70) | |
| sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:52) | |
| sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) | |
| sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) | |
| java.nio.file.Files.readAttributes(Files.java:1737) | |
| java.nio.file.Files.size(Files.java:2332) | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 4.6% (23.1ms out of 500ms) cpu usage by thread 'elasticsearch[arm-or-009_data][refresh][T#1]' | |
| 7/10 snapshots sharing following 37 elements | |
| sun.nio.fs.UnixNativeDispatcher.unlink0(Native Method) | |
| sun.nio.fs.UnixNativeDispatcher.unlink(UnixNativeDispatcher.java:141) | |
| sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:231) | |
| sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:103) | |
| java.nio.file.Files.delete(Files.java:1126) | |
| org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:373) | |
| org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:335) | |
| org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:62) | |
| org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:741) | |
| org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:746) | |
| org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) | |
| org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:724) | |
| org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:718) | |
| org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:594) | |
| org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:648) | |
| org.apache.lucene.index.IndexWriter.decRefDeleter(IndexWriter.java:5089) | |
| org.apache.lucene.index.StandardDirectoryReader.doClose(StandardDirectoryReader.java:383) | |
| org.apache.lucene.index.IndexReader.decRef(IndexReader.java:247) | |
| org.apache.lucene.index.IndexReader.close(IndexReader.java:393) | |
| org.apache.lucene.index.FilterDirectoryReader.doClose(FilterDirectoryReader.java:134) | |
| org.apache.lucene.index.IndexReader.decRef(IndexReader.java:247) | |
| org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.decRef(InternalEngine.java:330) | |
| org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.decRef(InternalEngine.java:275) | |
| org.apache.lucene.search.ReferenceManager.release(ReferenceManager.java:274) | |
| org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:189) | |
| org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1401) | |
| org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1383) | |
| org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:861) | |
| org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:698) | |
| org.elasticsearch.index.IndexService.access$400(IndexService.java:97) | |
| org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:902) | |
| org.elasticsearch.index.IndexService$BaseAsyncTask.run(IndexService.java:812) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 3/10 snapshots sharing following 24 elements | |
| org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293) | |
| org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268) | |
| org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258) | |
| org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104) | |
| org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140) | |
| org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:156) | |
| org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58) | |
| org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:300) | |
| org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:275) | |
| org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1401) | |
| org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1383) | |
| org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:861) | |
| org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:698) | |
| org.elasticsearch.index.IndexService.access$400(IndexService.java:97) | |
| org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:902) | |
| org.elasticsearch.index.IndexService$BaseAsyncTask.run(IndexService.java:812) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 3.6% (18.1ms out of 500ms) cpu usage by thread 'elasticsearch[arm-or-009_data][bulk][T#3]' | |
| 5/10 snapshots sharing following 47 elements | |
| sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| java.nio.channels.Channels.writeFully(Channels.java:101) | |
| java.nio.channels.Channels.access$000(Channels.java:61) | |
| java.nio.channels.Channels$1.write(Channels.java:174) | |
| java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| org.elasticsearch.common.bytes.BytesReference.writeTo(BytesReference.java:68) | |
| org.elasticsearch.index.translog.TranslogWriter.add(TranslogWriter.java:195) | |
| org.elasticsearch.index.translog.Translog.add(Translog.java:505) | |
| org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:886) | |
| org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:738) | |
| org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:707) | |
| org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:673) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.executeIndexRequestOnPrimary(TransportShardBulkAction.java:548) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.executeUpdateRequestOnce(TransportShardBulkAction.java:359) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.executeUpdateRequest(TransportShardBulkAction.java:406) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:240) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:123) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:110) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:72) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:1034) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:1012) | |
| org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:103) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:359) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:299) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:975) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:972) | |
| org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:238) | |
| org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationPermit(IndexShard.java:2220) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryShardReference(TransportReplicationAction.java:984) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction.access$500(TransportReplicationAction.java:98) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:320) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:295) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:282) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:656) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 4/10 snapshots sharing following 43 elements | |
| java.nio.Bits.copyToArray(Bits.java:835) | |
| java.nio.DirectByteBuffer.get(DirectByteBuffer.java:280) | |
| org.apache.lucene.store.ByteBufferGuard.getBytes(ByteBufferGuard.java:93) | |
| org.apache.lucene.store.ByteBufferIndexInput.readBytes(ByteBufferIndexInput.java:89) | |
| org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.loadBlock(SegmentTermsEnumFrame.java:173) | |
| org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:507) | |
| org.elasticsearch.common.lucene.uid.PerThreadIDVersionAndSeqNoLookup.getDocID(PerThreadIDVersionAndSeqNoLookup.java:114) | |
| org.elasticsearch.common.lucene.uid.PerThreadIDVersionAndSeqNoLookup.lookupVersion(PerThreadIDVersionAndSeqNoLookup.java:93) | |
| org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.loadDocIdAndVersion(VersionsAndSeqNoResolver.java:137) | |
| org.elasticsearch.index.engine.Engine.getFromSearcher(Engine.java:461) | |
| org.elasticsearch.index.engine.InternalEngine.get(InternalEngine.java:646) | |
| org.elasticsearch.index.shard.IndexShard.get(IndexShard.java:850) | |
| org.elasticsearch.index.get.ShardGetService.innerGet(ShardGetService.java:153) | |
| org.elasticsearch.index.get.ShardGetService.get(ShardGetService.java:81) | |
| org.elasticsearch.action.update.UpdateHelper.prepare(UpdateHelper.java:74) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.executeUpdateRequestOnce(TransportShardBulkAction.java:343) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.executeUpdateRequest(TransportShardBulkAction.java:406) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:240) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:123) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:110) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:72) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:1034) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:1012) | |
| org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:103) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:359) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:299) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:975) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:972) | |
| org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:238) | |
| org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationPermit(IndexShard.java:2220) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryShardReference(TransportReplicationAction.java:984) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction.access$500(TransportReplicationAction.java:98) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:320) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:295) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:282) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:656) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| unique snapshot | |
| java.nio.Bits.copyToArray(Bits.java:835) | |
| java.nio.DirectByteBuffer.get(DirectByteBuffer.java:280) | |
| org.apache.lucene.store.ByteBufferGuard.getBytes(ByteBufferGuard.java:93) | |
| org.apache.lucene.store.ByteBufferIndexInput.readBytes(ByteBufferIndexInput.java:89) | |
| org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.loadBlock(SegmentTermsEnumFrame.java:173) | |
| org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:507) | |
| org.elasticsearch.common.lucene.uid.PerThreadIDVersionAndSeqNoLookup.getDocID(PerThreadIDVersionAndSeqNoLookup.java:114) | |
| org.elasticsearch.common.lucene.uid.PerThreadIDVersionAndSeqNoLookup.lookupSeqNo(PerThreadIDVersionAndSeqNoLookup.java:134) | |
| org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.loadDocIdAndSeqNo(VersionsAndSeqNoResolver.java:159) | |
| org.elasticsearch.index.engine.InternalEngine.compareOpToLuceneDocBasedOnSeqNo(InternalEngine.java:679) | |
| org.elasticsearch.index.engine.InternalEngine.planIndexingAsNonPrimary(InternalEngine.java:943) | |
| org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:870) | |
| org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:738) | |
| org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:707) | |
| org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:680) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:518) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:480) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:466) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:72) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:567) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:530) | |
| org.elasticsearch.index.shard.IndexShard$2.onResponse(IndexShard.java:2314) | |
| org.elasticsearch.index.shard.IndexShard$2.onResponse(IndexShard.java:2292) | |
| org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:238) | |
| org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationPermit(IndexShard.java:2291) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:641) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:513) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:493) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| ::: {arm-or-003_data}{R_GaKyzFRV6xTimYa6PPrg}{G-7mXv6ARDW7fDI-PLWx9Q}{plxcq8199.myserver.com}{x.x.x.x:9300}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.410Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 95.4% (477.1ms out of 500ms) cpu usage by thread 'elasticsearch[arm-or-003_data][management][T#5]' | |
| 7/10 snapshots sharing following 37 elements | |
| java.io.UnixFileSystem.canonicalize0(Native Method) | |
| java.io.UnixFileSystem.canonicalize(UnixFileSystem.java:172) | |
| java.io.File.getCanonicalPath(File.java:618) | |
| java.io.FilePermission$1.run(FilePermission.java:215) | |
| java.io.FilePermission$1.run(FilePermission.java:203) | |
| java.security.AccessController.doPrivileged(Native Method) | |
| java.io.FilePermission.init(FilePermission.java:203) | |
| java.io.FilePermission.<init>(FilePermission.java:277) | |
| java.lang.SecurityManager.checkRead(SecurityManager.java:888) | |
| sun.nio.fs.UnixPath.checkRead(UnixPath.java:795) | |
| sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49) | |
| sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) | |
| sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) | |
| java.nio.file.Files.readAttributes(Files.java:1737) | |
| java.nio.file.Files.size(Files.java:2332) | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 3/10 snapshots sharing following 9 elements | |
| sun.misc.Unsafe.park(Native Method) | |
| java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) | |
| java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:734) | |
| java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647) | |
| java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1277) | |
| java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| ::: {arm-or-006_ingest}{DjAvKlWcRV-0imGYVyAhfQ}{QH012qOqS168FAL6CMZMQw}{plxcq8202.myserver.com}{x.x.x.x:9302}{box_type=hot, ibi_site=pdx} | |
| Hot threads at 2019-12-12T23:52:10.411Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| ::: {arm-or-005_master}{jydl9Gz2Shme0C9uSIAlDQ}{tr65OkiVSRCn8Ab5I4SdNQ}{plxcq8201.myserver.com}{x.x.x.x:9301}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.410Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 0.0% (86micros out of 500ms) cpu usage by thread 'elasticsearch[arm-or-005_master][[timer]]' | |
| 10/10 snapshots sharing following 2 elements | |
| java.lang.Thread.sleep(Native Method) | |
| org.elasticsearch.threadpool.ThreadPool$CachedTimeThread.run(ThreadPool.java:541) | |
| ::: {arm-or-010_ingest}{tGq_571FSkehKjbM_rHsUQ}{fRYKwhFNTMy0rm6xmi7j-w}{plxcq8206.myserver.com}{x.x.x.x:9302}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.413Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| ::: {arm-or-010_data}{_fIi48y9TUeeYMDgbqnEqA}{65exRIv9RbGfrk2u3DCcaQ}{plxcq8206.myserver.com}{x.x.x.x:9300}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.412Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 2.8% (14ms out of 500ms) cpu usage by thread 'elasticsearch[arm-or-010_data][refresh][T#1]' | |
| 8/10 snapshots sharing following 2 elements | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 1.6% (7.7ms out of 500ms) cpu usage by thread 'elasticsearch[arm-or-010_data][bulk][T#1]' | |
| 4/10 snapshots sharing following 2 elements | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| ::: {arm-or-007_ingest}{fERzhejuRE6_MCMshvcArw}{RDh8CVPRQUWh21scNnVNtA}{plxcq8203.myserver.com}{x.x.x.x:9302}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.411Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| ::: {arm-or-008_ingest}{VAE3rT3gT-OABoQjPeo4BQ}{RtZkgrsUR6C610e8TX0dhw}{plxcq8204.myserver.com}{x.x.x.x:9302}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.413Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| ::: {arm-or-002_data}{WYL2OYuhQkGRzzsCD1XSUQ}{7pNJCaIURrK42LYSy1QWZA}{plxcq8198.myserver.com}{x.x.x.x:9300}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.412Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 88.9% (444.5ms out of 500ms) cpu usage by thread 'elasticsearch[arm-or-002_data][management][T#5]' | |
| 8/10 snapshots sharing following 37 elements | |
| java.io.UnixFileSystem.canonicalize0(Native Method) | |
| java.io.UnixFileSystem.canonicalize(UnixFileSystem.java:172) | |
| java.io.File.getCanonicalPath(File.java:618) | |
| java.io.FilePermission$1.run(FilePermission.java:215) | |
| java.io.FilePermission$1.run(FilePermission.java:203) | |
| java.security.AccessController.doPrivileged(Native Method) | |
| java.io.FilePermission.init(FilePermission.java:203) | |
| java.io.FilePermission.<init>(FilePermission.java:277) | |
| java.lang.SecurityManager.checkRead(SecurityManager.java:888) | |
| sun.nio.fs.UnixPath.checkRead(UnixPath.java:795) | |
| sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49) | |
| sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) | |
| sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) | |
| java.nio.file.Files.readAttributes(Files.java:1737) | |
| java.nio.file.Files.size(Files.java:2332) | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 2/10 snapshots sharing following 32 elements | |
| java.security.AccessController.getStackAccessControlContext(Native Method) | |
| java.security.AccessController.checkPermission(AccessController.java:858) | |
| java.lang.SecurityManager.checkPermission(SecurityManager.java:549) | |
| java.lang.SecurityManager.checkRead(SecurityManager.java:888) | |
| sun.nio.fs.UnixPath.checkRead(UnixPath.java:795) | |
| sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49) | |
| sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) | |
| sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) | |
| java.nio.file.Files.readAttributes(Files.java:1737) | |
| java.nio.file.Files.size(Files.java:2332) | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 6.4% (31.9ms out of 500ms) cpu usage by thread 'elasticsearch[arm-or-002_data][refresh][T#2]' | |
| 3/10 snapshots sharing following 37 elements | |
| sun.nio.fs.UnixNativeDispatcher.unlink0(Native Method) | |
| sun.nio.fs.UnixNativeDispatcher.unlink(UnixNativeDispatcher.java:141) | |
| sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:231) | |
| sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:103) | |
| java.nio.file.Files.delete(Files.java:1126) | |
| org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:373) | |
| org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:335) | |
| org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:62) | |
| org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:741) | |
| org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:746) | |
| org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) | |
| org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:724) | |
| org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:718) | |
| org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:594) | |
| org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:648) | |
| org.apache.lucene.index.IndexWriter.decRefDeleter(IndexWriter.java:5089) | |
| org.apache.lucene.index.StandardDirectoryReader.doClose(StandardDirectoryReader.java:383) | |
| org.apache.lucene.index.IndexReader.decRef(IndexReader.java:247) | |
| org.apache.lucene.index.IndexReader.close(IndexReader.java:393) | |
| org.apache.lucene.index.FilterDirectoryReader.doClose(FilterDirectoryReader.java:134) | |
| org.apache.lucene.index.IndexReader.decRef(IndexReader.java:247) | |
| org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.decRef(InternalEngine.java:330) | |
| org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.decRef(InternalEngine.java:275) | |
| org.apache.lucene.search.ReferenceManager.release(ReferenceManager.java:274) | |
| org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:189) | |
| org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1401) | |
| org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1383) | |
| org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:861) | |
| org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:698) | |
| org.elasticsearch.index.IndexService.access$400(IndexService.java:97) | |
| org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:902) | |
| org.elasticsearch.index.IndexService$BaseAsyncTask.run(IndexService.java:812) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 2/10 snapshots sharing following 45 elements | |
| java.io.UnixFileSystem.canonicalize0(Native Method) | |
| java.io.UnixFileSystem.canonicalize(UnixFileSystem.java:172) | |
| java.io.File.getCanonicalPath(File.java:618) | |
| java.io.FilePermission$1.run(FilePermission.java:215) | |
| java.io.FilePermission$1.run(FilePermission.java:203) | |
| java.security.AccessController.doPrivileged(Native Method) | |
| java.io.FilePermission.init(FilePermission.java:203) | |
| java.io.FilePermission.<init>(FilePermission.java:277) | |
| java.lang.SecurityManager.checkDelete(SecurityManager.java:1007) | |
| sun.nio.fs.UnixPath.checkDelete(UnixPath.java:807) | |
| sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:222) | |
| sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:103) | |
| java.nio.file.Files.delete(Files.java:1126) | |
| org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:373) | |
| org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:335) | |
| org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:62) | |
| org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:741) | |
| org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:746) | |
| org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) | |
| org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:724) | |
| org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:718) | |
| org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:594) | |
| org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:648) | |
| org.apache.lucene.index.IndexWriter.decRefDeleter(IndexWriter.java:5089) | |
| org.apache.lucene.index.StandardDirectoryReader.doClose(StandardDirectoryReader.java:383) | |
| org.apache.lucene.index.IndexReader.decRef(IndexReader.java:247) | |
| org.apache.lucene.index.IndexReader.close(IndexReader.java:393) | |
| org.apache.lucene.index.FilterDirectoryReader.doClose(FilterDirectoryReader.java:134) | |
| org.apache.lucene.index.IndexReader.decRef(IndexReader.java:247) | |
| org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.decRef(InternalEngine.java:330) | |
| org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.decRef(InternalEngine.java:275) | |
| org.apache.lucene.search.ReferenceManager.release(ReferenceManager.java:274) | |
| org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:189) | |
| org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1401) | |
| org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1383) | |
| org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:861) | |
| org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:698) | |
| org.elasticsearch.index.IndexService.access$400(IndexService.java:97) | |
| org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:902) | |
| org.elasticsearch.index.IndexService$BaseAsyncTask.run(IndexService.java:812) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 2/10 snapshots sharing following 28 elements | |
| org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:491) | |
| org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556) | |
| org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:672) | |
| org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:453) | |
| org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293) | |
| org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268) | |
| org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258) | |
| org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104) | |
| org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140) | |
| org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:156) | |
| org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58) | |
| org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:300) | |
| org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:275) | |
| org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1401) | |
| org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1383) | |
| org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:861) | |
| org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:698) | |
| org.elasticsearch.index.IndexService.access$400(IndexService.java:97) | |
| org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:902) | |
| org.elasticsearch.index.IndexService$BaseAsyncTask.run(IndexService.java:812) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 2/10 snapshots sharing following 24 elements | |
| org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293) | |
| org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268) | |
| org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258) | |
| org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104) | |
| org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140) | |
| org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:156) | |
| org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58) | |
| org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:300) | |
| org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:275) | |
| org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1401) | |
| org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1383) | |
| org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:861) | |
| org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:698) | |
| org.elasticsearch.index.IndexService.access$400(IndexService.java:97) | |
| org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:902) | |
| org.elasticsearch.index.IndexService$BaseAsyncTask.run(IndexService.java:812) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| unique snapshot | |
| sun.misc.Unsafe.park(Native Method) | |
| java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) | |
| java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:734) | |
| java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647) | |
| java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1277) | |
| java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| ::: {arm-or-005_data}{RqxttAceQFW1XC9tquQIWQ}{L5ap5XoXRsWF5W7Xq6SKaQ}{plxcq8201.myserver.com}{x.x.x.x:9300}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.409Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 72.1% (360.3ms out of 500ms) cpu usage by thread 'elasticsearch[arm-or-005_data][management][T#3]' | |
| 6/10 snapshots sharing following 37 elements | |
| java.io.UnixFileSystem.canonicalize0(Native Method) | |
| java.io.UnixFileSystem.canonicalize(UnixFileSystem.java:172) | |
| java.io.File.getCanonicalPath(File.java:618) | |
| java.io.FilePermission$1.run(FilePermission.java:215) | |
| java.io.FilePermission$1.run(FilePermission.java:203) | |
| java.security.AccessController.doPrivileged(Native Method) | |
| java.io.FilePermission.init(FilePermission.java:203) | |
| java.io.FilePermission.<init>(FilePermission.java:277) | |
| java.lang.SecurityManager.checkRead(SecurityManager.java:888) | |
| sun.nio.fs.UnixPath.checkRead(UnixPath.java:795) | |
| sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49) | |
| sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) | |
| sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) | |
| java.nio.file.Files.readAttributes(Files.java:1737) | |
| java.nio.file.Files.size(Files.java:2332) | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 3/10 snapshots sharing following 30 elements | |
| sun.nio.fs.UnixNativeDispatcher.stat0(Native Method) | |
| sun.nio.fs.UnixNativeDispatcher.stat(UnixNativeDispatcher.java:286) | |
| sun.nio.fs.UnixFileAttributes.get(UnixFileAttributes.java:70) | |
| sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:52) | |
| sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) | |
| sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) | |
| java.nio.file.Files.readAttributes(Files.java:1737) | |
| java.nio.file.Files.size(Files.java:2332) | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| unique snapshot | |
| java.security.AccessController.getStackAccessControlContext(Native Method) | |
| java.security.AccessController.checkPermission(AccessController.java:858) | |
| java.lang.SecurityManager.checkPermission(SecurityManager.java:549) | |
| java.lang.SecurityManager.checkRead(SecurityManager.java:888) | |
| sun.nio.fs.UnixPath.checkRead(UnixPath.java:795) | |
| sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49) | |
| sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) | |
| sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) | |
| java.nio.file.Files.readAttributes(Files.java:1737) | |
| java.nio.file.Files.size(Files.java:2332) | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| ::: {arm-or-001_data}{qN88akkuQQ6F90isVQ6hxQ}{oXbmjRd9QJayW8_M8M0U9w}{plxcq8197.myserver.com}{x.x.x.x:9300}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.411Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 82.1% (410.4ms out of 500ms) cpu usage by thread 'elasticsearch[arm-or-001_data][management][T#5]' | |
| 9/10 snapshots sharing following 32 elements | |
| java.security.AccessController.doPrivileged(Native Method) | |
| java.io.FilePermission.init(FilePermission.java:203) | |
| java.io.FilePermission.<init>(FilePermission.java:277) | |
| java.lang.SecurityManager.checkRead(SecurityManager.java:888) | |
| sun.nio.fs.UnixPath.checkRead(UnixPath.java:795) | |
| sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49) | |
| sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) | |
| sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) | |
| java.nio.file.Files.readAttributes(Files.java:1737) | |
| java.nio.file.Files.size(Files.java:2332) | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| unique snapshot | |
| java.security.AccessController.getStackAccessControlContext(Native Method) | |
| java.security.AccessController.checkPermission(AccessController.java:858) | |
| java.lang.SecurityManager.checkPermission(SecurityManager.java:549) | |
| java.lang.SecurityManager.checkRead(SecurityManager.java:888) | |
| sun.nio.fs.UnixPath.checkRead(UnixPath.java:795) | |
| sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49) | |
| sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) | |
| sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) | |
| java.nio.file.Files.readAttributes(Files.java:1737) | |
| java.nio.file.Files.size(Files.java:2332) | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| ::: {arm-or-008_data}{GGpw8eA5RpKJmLAC5dqyiQ}{_DLc3WiKSgCpdW432t7tgg}{plxcq8204.myserver.com}{x.x.x.x:9300}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.413Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 75.2% (376ms out of 500ms) cpu usage by thread 'elasticsearch[arm-or-008_data][management][T#2]' | |
| 6/10 snapshots sharing following 37 elements | |
| java.io.UnixFileSystem.canonicalize0(Native Method) | |
| java.io.UnixFileSystem.canonicalize(UnixFileSystem.java:172) | |
| java.io.File.getCanonicalPath(File.java:618) | |
| java.io.FilePermission$1.run(FilePermission.java:215) | |
| java.io.FilePermission$1.run(FilePermission.java:203) | |
| java.security.AccessController.doPrivileged(Native Method) | |
| java.io.FilePermission.init(FilePermission.java:203) | |
| java.io.FilePermission.<init>(FilePermission.java:277) | |
| java.lang.SecurityManager.checkRead(SecurityManager.java:888) | |
| sun.nio.fs.UnixPath.checkRead(UnixPath.java:795) | |
| sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49) | |
| sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) | |
| sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) | |
| java.nio.file.Files.readAttributes(Files.java:1737) | |
| java.nio.file.Files.size(Files.java:2332) | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 2/10 snapshots sharing following 22 elements | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 2/10 snapshots sharing following 13 elements | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| ::: {arm-or-002_master}{qm2d5LDBQey92KShf2M2-A}{5E2xkmZIRj2mzXFIPgOkQg}{plxcq8198.myserver.com}{x.x.x.x:9301}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.413Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 0.0% (69.9micros out of 500ms) cpu usage by thread 'elasticsearch[arm-or-002_master][[timer]]' | |
| 10/10 snapshots sharing following 2 elements | |
| java.lang.Thread.sleep(Native Method) | |
| org.elasticsearch.threadpool.ThreadPool$CachedTimeThread.run(ThreadPool.java:541) | |
| ::: {arm-or-004_data}{qaQWdMVpToOq-18Ty2waaA}{zZjOUixCQG-4lSeRMw4XvQ}{plxcq8200.myserver.com}{x.x.x.x:9300}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.411Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 85.0% (425ms out of 500ms) cpu usage by thread 'elasticsearch[arm-or-004_data][management][T#5]' | |
| 5/10 snapshots sharing following 37 elements | |
| java.io.UnixFileSystem.canonicalize0(Native Method) | |
| java.io.UnixFileSystem.canonicalize(UnixFileSystem.java:172) | |
| java.io.File.getCanonicalPath(File.java:618) | |
| java.io.FilePermission$1.run(FilePermission.java:215) | |
| java.io.FilePermission$1.run(FilePermission.java:203) | |
| java.security.AccessController.doPrivileged(Native Method) | |
| java.io.FilePermission.init(FilePermission.java:203) | |
| java.io.FilePermission.<init>(FilePermission.java:277) | |
| java.lang.SecurityManager.checkRead(SecurityManager.java:888) | |
| sun.nio.fs.UnixPath.checkRead(UnixPath.java:795) | |
| sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49) | |
| sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) | |
| sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) | |
| java.nio.file.Files.readAttributes(Files.java:1737) | |
| java.nio.file.Files.size(Files.java:2332) | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 2/10 snapshots sharing following 30 elements | |
| sun.nio.fs.UnixNativeDispatcher.stat0(Native Method) | |
| sun.nio.fs.UnixNativeDispatcher.stat(UnixNativeDispatcher.java:286) | |
| sun.nio.fs.UnixFileAttributes.get(UnixFileAttributes.java:70) | |
| sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:52) | |
| sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) | |
| sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) | |
| java.nio.file.Files.readAttributes(Files.java:1737) | |
| java.nio.file.Files.size(Files.java:2332) | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 3/10 snapshots sharing following 9 elements | |
| sun.misc.Unsafe.park(Native Method) | |
| java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) | |
| java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:734) | |
| java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647) | |
| java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1277) | |
| java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| ::: {arm-or-004_master}{rzsh0R3LRxOeGeTp3BwTbA}{YuzXQ2skQxOxyIjKskxVkw}{plxcq8200.myserver.com}{x.x.x.x:9301}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.411Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 0.0% (55.6micros out of 500ms) cpu usage by thread 'elasticsearch[arm-or-004_master][[timer]]' | |
| 10/10 snapshots sharing following 2 elements | |
| java.lang.Thread.sleep(Native Method) | |
| org.elasticsearch.threadpool.ThreadPool$CachedTimeThread.run(ThreadPool.java:541) | |
| ::: {arm-or-006_data}{h7SOKT2mQAqkt91vHZg0rQ}{Csa_teMERI-kMaCImHJTjQ}{plxcq8202.myserver.com}{x.x.x.x:9300}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.543Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 53.9% (269.4ms out of 500ms) cpu usage by thread 'elasticsearch[arm-or-006_data][management][T#1]' | |
| 2/10 snapshots sharing following 32 elements | |
| java.security.AccessController.getStackAccessControlContext(Native Method) | |
| java.security.AccessController.checkPermission(AccessController.java:858) | |
| java.lang.SecurityManager.checkPermission(SecurityManager.java:549) | |
| java.lang.SecurityManager.checkRead(SecurityManager.java:888) | |
| sun.nio.fs.UnixPath.checkRead(UnixPath.java:795) | |
| sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49) | |
| sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) | |
| sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) | |
| java.nio.file.Files.readAttributes(Files.java:1737) | |
| java.nio.file.Files.size(Files.java:2332) | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 8/10 snapshots sharing following 19 elements | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 9.5% (47.3ms out of 500ms) cpu usage by thread 'elasticsearch[arm-or-006_data][bulk][T#3]' | |
| 2/10 snapshots sharing following 46 elements | |
| org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:556) | |
| org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:672) | |
| org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:453) | |
| org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293) | |
| org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268) | |
| org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258) | |
| org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104) | |
| org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140) | |
| org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:156) | |
| org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58) | |
| org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1405) | |
| org.elasticsearch.index.engine.InternalEngine.get(InternalEngine.java:637) | |
| org.elasticsearch.index.shard.IndexShard.get(IndexShard.java:850) | |
| org.elasticsearch.index.get.ShardGetService.innerGet(ShardGetService.java:153) | |
| org.elasticsearch.index.get.ShardGetService.get(ShardGetService.java:81) | |
| org.elasticsearch.action.update.UpdateHelper.prepare(UpdateHelper.java:74) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.executeUpdateRequestOnce(TransportShardBulkAction.java:343) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.executeUpdateRequest(TransportShardBulkAction.java:406) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:240) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:123) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:110) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:72) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:1034) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:1012) | |
| org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:103) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:359) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:299) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:975) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:972) | |
| org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:238) | |
| org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationPermit(IndexShard.java:2220) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryShardReference(TransportReplicationAction.java:984) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction.access$500(TransportReplicationAction.java:98) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:320) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:295) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:282) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:656) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 4/10 snapshots sharing following 32 elements | |
| org.elasticsearch.index.shard.IndexShard.get(IndexShard.java:850) | |
| org.elasticsearch.index.get.ShardGetService.innerGet(ShardGetService.java:153) | |
| org.elasticsearch.index.get.ShardGetService.get(ShardGetService.java:81) | |
| org.elasticsearch.action.update.UpdateHelper.prepare(UpdateHelper.java:74) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.executeUpdateRequestOnce(TransportShardBulkAction.java:343) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.executeUpdateRequest(TransportShardBulkAction.java:406) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:240) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:123) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:110) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:72) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:1034) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:1012) | |
| org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:103) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:359) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:299) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:975) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:972) | |
| org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:238) | |
| org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationPermit(IndexShard.java:2220) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryShardReference(TransportReplicationAction.java:984) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction.access$500(TransportReplicationAction.java:98) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:320) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:295) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:282) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:656) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 4/10 snapshots sharing following 36 elements | |
| java.nio.Bits.copyToArray(Bits.java:835) | |
| java.nio.DirectByteBuffer.get(DirectByteBuffer.java:280) | |
| org.apache.lucene.store.ByteBufferGuard.getBytes(ByteBufferGuard.java:93) | |
| org.apache.lucene.store.ByteBufferIndexInput.readBytes(ByteBufferIndexInput.java:89) | |
| org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.loadBlock(SegmentTermsEnumFrame.java:173) | |
| org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:507) | |
| org.elasticsearch.common.lucene.uid.PerThreadIDVersionAndSeqNoLookup.getDocID(PerThreadIDVersionAndSeqNoLookup.java:114) | |
| org.elasticsearch.common.lucene.uid.PerThreadIDVersionAndSeqNoLookup.lookupSeqNo(PerThreadIDVersionAndSeqNoLookup.java:134) | |
| org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.loadDocIdAndSeqNo(VersionsAndSeqNoResolver.java:159) | |
| org.elasticsearch.index.engine.InternalEngine.compareOpToLuceneDocBasedOnSeqNo(InternalEngine.java:679) | |
| org.elasticsearch.index.engine.InternalEngine.planIndexingAsNonPrimary(InternalEngine.java:943) | |
| org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:870) | |
| org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:738) | |
| org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:707) | |
| org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:680) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:518) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:480) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:466) | |
| org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:72) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:567) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:530) | |
| org.elasticsearch.index.shard.IndexShard$2.onResponse(IndexShard.java:2314) | |
| org.elasticsearch.index.shard.IndexShard$2.onResponse(IndexShard.java:2292) | |
| org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:238) | |
| org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationPermit(IndexShard.java:2291) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:641) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:513) | |
| org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:493) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 9.1% (45.2ms out of 500ms) cpu usage by thread 'elasticsearch[arm-or-006_data][refresh][T#1]' | |
| 2/10 snapshots sharing following 28 elements | |
| org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:208) | |
| org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:258) | |
| org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:105) | |
| org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:490) | |
| org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293) | |
| org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:268) | |
| org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:258) | |
| org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104) | |
| org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140) | |
| org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:156) | |
| org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58) | |
| org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:300) | |
| org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:275) | |
| org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1401) | |
| org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1383) | |
| org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:861) | |
| org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:698) | |
| org.elasticsearch.index.IndexService.access$400(IndexService.java:97) | |
| org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:902) | |
| org.elasticsearch.index.IndexService$BaseAsyncTask.run(IndexService.java:812) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 8/10 snapshots sharing following 2 elements | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| ::: {arm-or-009_ingest}{a5voC7oXSf-78LfPlJ45Yg}{YHB_yBFTQ7OuqqPMiyuowQ}{plxcq8205.myserver.com}{x.x.x.x:9302}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.410Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 0.0% (36.6micros out of 500ms) cpu usage by thread 'elasticsearch[arm-or-009_ingest][[timer]]' | |
| 10/10 snapshots sharing following 2 elements | |
| java.lang.Thread.sleep(Native Method) | |
| org.elasticsearch.threadpool.ThreadPool$CachedTimeThread.run(ThreadPool.java:541) | |
| ::: {arm-or-003_master}{9WloaQB3RRS2keOlRB1hzA}{qJHKAqZqQ429ANxdn1UUKA}{plxcq8199.myserver.com}{x.x.x.x:9301}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.411Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 0.6% (3.2ms out of 500ms) cpu usage by thread 'elasticsearch[arm-or-003_master][management][T#3]' | |
| 10/10 snapshots sharing following 14 elements | |
| sun.misc.Unsafe.park(Native Method) | |
| java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) | |
| java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) | |
| java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) | |
| java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) | |
| org.elasticsearch.cluster.InternalClusterInfoService.refresh(InternalClusterInfoService.java:357) | |
| org.elasticsearch.cluster.InternalClusterInfoService.maybeRefresh(InternalClusterInfoService.java:275) | |
| org.elasticsearch.cluster.InternalClusterInfoService.access$500(InternalClusterInfoService.java:68) | |
| org.elasticsearch.cluster.InternalClusterInfoService$SubmitReschedulingClusterInfoUpdatedJob.lambda$run$0(InternalClusterInfoService.java:222) | |
| org.elasticsearch.cluster.InternalClusterInfoService$SubmitReschedulingClusterInfoUpdatedJob$$Lambda$2374/861842000.run(Unknown Source) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| ::: {arm-or-007_data}{9zEBQRPGTpSA_plraQfAMA}{ldXxb_vgT2ymHGKq8OoxBw}{plxcq8203.myserver.com}{x.x.x.x:9300}{ibi_site=pdx, box_type=hot} | |
| Hot threads at 2019-12-12T23:52:10.411Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true: | |
| 84.4% (422.2ms out of 500ms) cpu usage by thread 'elasticsearch[arm-or-007_data][management][T#5]' | |
| 8/10 snapshots sharing following 37 elements | |
| java.io.UnixFileSystem.canonicalize0(Native Method) | |
| java.io.UnixFileSystem.canonicalize(UnixFileSystem.java:172) | |
| java.io.File.getCanonicalPath(File.java:618) | |
| java.io.FilePermission$1.run(FilePermission.java:215) | |
| java.io.FilePermission$1.run(FilePermission.java:203) | |
| java.security.AccessController.doPrivileged(Native Method) | |
| java.io.FilePermission.init(FilePermission.java:203) | |
| java.io.FilePermission.<init>(FilePermission.java:277) | |
| java.lang.SecurityManager.checkRead(SecurityManager.java:888) | |
| sun.nio.fs.UnixPath.checkRead(UnixPath.java:795) | |
| sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49) | |
| sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) | |
| sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) | |
| java.nio.file.Files.readAttributes(Files.java:1737) | |
| java.nio.file.Files.size(Files.java:2332) | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
| 2/10 snapshots sharing following 26 elements | |
| sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) | |
| sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) | |
| java.nio.file.Files.readAttributes(Files.java:1737) | |
| java.nio.file.Files.size(Files.java:2332) | |
| org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243) | |
| org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:67) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.estimateSize(Store.java:1448) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1437) | |
| org.elasticsearch.index.store.Store$StoreStatsCache.refresh(Store.java:1426) | |
| org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) | |
| org.elasticsearch.index.store.Store.stats(Store.java:375) | |
| org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:953) | |
| org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:178) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:164) | |
| org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:45) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:433) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:412) | |
| org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:399) | |
| org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30) | |
| org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) | |
| org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) | |
| org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) | |
| org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| java.lang.Thread.run(Thread.java:748) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment