gistfile1.txt

::: [log1-02][oJeslZxkSSuOZ1fZ4lXA6Q][host][inet[/10.212.x.x:9300]] 0.6% (2.7ms out of 500ms) cpu usage by thread 'elasticsearch[log1-02][bulk][T#1]' 10/10 snapshots sharing following 10 elements sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735) java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644) java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137) org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) 0.1% (679.3micros out of 500ms) cpu usage by thread 'elasticsearch[log1-02][bulk][T#5]' 10/10 snapshots sharing following 10 elements sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735) java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644) java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137) org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) 0.1% (274micros out of 500ms) cpu usage by thread 'elasticsearch[log1-02][[transport_server_worker.default]][T#27]{New I/O worker #60}' 10/10 snapshots sharing following 15 elements sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415) org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745)
::: [log1-01][MVQNSVV-T2m64zU_vvLVYg][host][inet[/10.212.x.x:9300]] 100.6% (502.9ms out of 500ms) cpu usage by thread 'elasticsearch[log1-01][listener][T#5]' 2/10 snapshots sharing following 24 elements org.elasticsearch.common.xcontent.XContentBuilder.writeMap(XContentBuilder.java:1157) org.elasticsearch.common.xcontent.XContentBuilder.writeValue(XContentBuilder.java:1197) org.elasticsearch.common.xcontent.XContentBuilder.writeMap(XContentBuilder.java:1163) org.elasticsearch.common.xcontent.XContentBuilder.writeValue(XContentBuilder.java:1197) org.elasticsearch.common.xcontent.XContentBuilder.writeMap(XContentBuilder.java:1163) org.elasticsearch.common.xcontent.XContentBuilder.writeValue(XContentBuilder.java:1197) org.elasticsearch.common.xcontent.XContentBuilder.writeMap(XContentBuilder.java:1163) org.elasticsearch.common.xcontent.XContentBuilder.writeValue(XContentBuilder.java:1197) org.elasticsearch.common.xcontent.XContentBuilder.writeMap(XContentBuilder.java:1163) org.elasticsearch.common.xcontent.XContentBuilder.writeValue(XContentBuilder.java:1197) org.elasticsearch.common.xcontent.XContentBuilder.writeMap(XContentBuilder.java:1163) org.elasticsearch.common.xcontent.XContentBuilder.map(XContentBuilder.java:1072) org.elasticsearch.rest.action.admin.indices.get.RestGetIndicesAction$1.writeMappings(RestGetIndicesAction.java:137) org.elasticsearch.rest.action.admin.indices.get.RestGetIndicesAction$1.buildResponse(RestGetIndicesAction.java:102) org.elasticsearch.rest.action.admin.indices.get.RestGetIndicesAction$1.buildResponse(RestGetIndicesAction.java:85) org.elasticsearch.rest.action.support.RestBuilderListener.buildResponse(RestBuilderListener.java:37) org.elasticsearch.rest.action.support.RestResponseListener.processResponse(RestResponseListener.java:43) org.elasticsearch.rest.action.support.RestActionListener.onResponse(RestActionListener.java:49) org.elasticsearch.client.support.AbstractIndicesAdminClient$1.onResponse(AbstractIndicesAdminClient.java:254) org.elasticsearch.client.support.AbstractIndicesAdminClient$1.onResponse(AbstractIndicesAdminClient.java:250) org.elasticsearch.action.support.TransportAction$ThreadedActionListener$1.run(TransportAction.java:113) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) 6/10 snapshots sharing following 22 elements org.elasticsearch.common.xcontent.XContentBuilder.writeMap(XContentBuilder.java:1163) org.elasticsearch.common.xcontent.XContentBuilder.writeValue(XContentBuilder.java:1197) org.elasticsearch.common.xcontent.XContentBuilder.writeMap(XContentBuilder.java:1163) org.elasticsearch.common.xcontent.XContentBuilder.writeValue(XContentBuilder.java:1197) org.elasticsearch.common.xcontent.XContentBuilder.writeMap(XContentBuilder.java:1163) org.elasticsearch.common.xcontent.XContentBuilder.writeValue(XContentBuilder.java:1197) org.elasticsearch.common.xcontent.XContentBuilder.writeMap(XContentBuilder.java:1163) org.elasticsearch.common.xcontent.XContentBuilder.writeValue(XContentBuilder.java:1197) org.elasticsearch.common.xcontent.XContentBuilder.writeMap(XContentBuilder.java:1163) org.elasticsearch.common.xcontent.XContentBuilder.map(XContentBuilder.java:1072) org.elasticsearch.rest.action.admin.indices.get.RestGetIndicesAction$1.writeMappings(RestGetIndicesAction.java:137) org.elasticsearch.rest.action.admin.indices.get.RestGetIndicesAction$1.buildResponse(RestGetIndicesAction.java:102) org.elasticsearch.rest.action.admin.indices.get.RestGetIndicesAction$1.buildResponse(RestGetIndicesAction.java:85) org.elasticsearch.rest.action.support.RestBuilderListener.buildResponse(RestBuilderListener.java:37) org.elasticsearch.rest.action.support.RestResponseListener.processResponse(RestResponseListener.java:43) org.elasticsearch.rest.action.support.RestActionListener.onResponse(RestActionListener.java:49) org.elasticsearch.client.support.AbstractIndicesAdminClient$1.onResponse(AbstractIndicesAdminClient.java:254) org.elasticsearch.client.support.AbstractIndicesAdminClient$1.onResponse(AbstractIndicesAdminClient.java:250) org.elasticsearch.action.support.TransportAction$ThreadedActionListener$1.run(TransportAction.java:113) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) 2/10 snapshots sharing following 19 elements org.elasticsearch.common.xcontent.XContentBuilder.writeValue(XContentBuilder.java:1197) org.elasticsearch.common.xcontent.XContentBuilder.writeMap(XContentBuilder.java:1163) org.elasticsearch.common.xcontent.XContentBuilder.writeValue(XContentBuilder.java:1197) org.elasticsearch.common.xcontent.XContentBuilder.writeMap(XContentBuilder.java:1163) org.elasticsearch.common.xcontent.XContentBuilder.writeValue(XContentBuilder.java:1197) org.elasticsearch.common.xcontent.XContentBuilder.writeMap(XContentBuilder.java:1163) org.elasticsearch.common.xcontent.XContentBuilder.map(XContentBuilder.java:1072) org.elasticsearch.rest.action.admin.indices.get.RestGetIndicesAction$1.writeMappings(RestGetIndicesAction.java:137) org.elasticsearch.rest.action.admin.indices.get.RestGetIndicesAction$1.buildResponse(RestGetIndicesAction.java:102) org.elasticsearch.rest.action.admin.indices.get.RestGetIndicesAction$1.buildResponse(RestGetIndicesAction.java:85) org.elasticsearch.rest.action.support.RestBuilderListener.buildResponse(RestBuilderListener.java:37) org.elasticsearch.rest.action.support.RestResponseListener.processResponse(RestResponseListener.java:43) org.elasticsearch.rest.action.support.RestActionListener.onResponse(RestActionListener.java:49) org.elasticsearch.client.support.AbstractIndicesAdminClient$1.onResponse(AbstractIndicesAdminClient.java:254) org.elasticsearch.client.support.AbstractIndicesAdminClient$1.onResponse(AbstractIndicesAdminClient.java:250) org.elasticsearch.action.support.TransportAction$ThreadedActionListener$1.run(TransportAction.java:113) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) 8.0% (40.2ms out of 500ms) cpu usage by thread 'elasticsearch[log1-01][[recovery_stream]][T#49]' 8/10 snapshots sharing following 13 elements sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) org.elasticsearch.common.util.concurrent.BaseFuture$Sync.get(BaseFuture.java:274) org.elasticsearch.common.util.concurrent.BaseFuture.get(BaseFuture.java:113) org.elasticsearch.transport.PlainTransportFuture.txGet(PlainTransportFuture.java:45) org.elasticsearch.transport.PlainTransportFuture.txGet(PlainTransportFuture.java:34) org.elasticsearch.indices.recovery.RecoverySource$1$1.run(RecoverySource.java:227) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) 2/10 snapshots sharing following 3 elements java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) 7.2% (36.1ms out of 500ms) cpu usage by thread 'elasticsearch[log1-01][[recovery_stream]][T#48]' 8/10 snapshots sharing following 13 elements sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) org.elasticsearch.common.util.concurrent.BaseFuture$Sync.get(BaseFuture.java:274) org.elasticsearch.common.util.concurrent.BaseFuture.get(BaseFuture.java:113) org.elasticsearch.transport.PlainTransportFuture.txGet(PlainTransportFuture.java:45) org.elasticsearch.transport.PlainTransportFuture.txGet(PlainTransportFuture.java:34) org.elasticsearch.indices.recovery.RecoverySource$1$1.run(RecoverySource.java:227) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) 2/10 snapshots sharing following 3 elements java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) 
::: [log1-04][f_eVPQPGS_25nucDk_jUwQ][host][inet[/10.212.x.x:9301]]{master=false} 1.2% (5.7ms out of 500ms) cpu usage by thread 'elasticsearch[log1-04][bulk][T#6]' 10/10 snapshots sharing following 10 elements sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735) java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644) java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137) org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) 1.0% (4.9ms out of 500ms) cpu usage by thread 'elasticsearch[log1-04][bulk][T#5]' 10/10 snapshots sharing following 10 elements sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735) java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644) java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137) org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) 0.6% (2.7ms out of 500ms) cpu usage by thread 'elasticsearch[log1-04][bulk][T#11]' 10/10 snapshots sharing following 10 elements sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735) java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644) java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137) org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) 
::: [log1-03][WkeNqdkTRXGDRDcmVF2YLw][host][inet[/10.212.x.x:9301]]{master=false} 11.3% (56.5ms out of 500ms) cpu usage by thread 'elasticsearch[log1-03][refresh][T#1]' 10/10 snapshots sharing following 9 elements sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735) java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644) java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) 4.9% (24.3ms out of 500ms) cpu usage by thread 'elasticsearch[log1-03][[transport_server_worker.default]][T#1]{New I/O worker #34}' 8/10 snapshots sharing following 15 elements sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415) org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) 2/10 snapshots sharing following 18 elements org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74) org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) 4.2% (20.8ms out of 500ms) cpu usage by thread 'elasticsearch[log1-03][[transport_server_worker.default]][T#2]{New I/O worker #35}' 10/10 snapshots sharing following 15 elements sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415) org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) "