Skip to content

Instantly share code, notes, and snippets.

@shardnit
Created July 31, 2012 11:01
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save shardnit/3216198 to your computer and use it in GitHub Desktop.
Save shardnit/3216198 to your computer and use it in GitHub Desktop.
ES Node2 jstack
2012-07-31 12:49:17
Full thread dump OpenJDK 64-Bit Server VM (20.0-b12 mixed mode):
"elasticsearch[node2][bulk][T#747]" daemon prio=10 tid=0x00007f5e30828000 nid=0x6e64 waiting on condition [0x00007f5e34d09000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][[index_with_metrics][2]: Lucene Merge Thread #5694]" daemon prio=10 tid=0x00007f5e0885c000 nid=0x6e63 runnable [0x00007f5e364a7000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.NativeThread.current(Native Method)
at sun.nio.ch.NativeThreadSet.add(NativeThreadSet.java:45)
at sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:661)
at org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:162)
at org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:270)
at org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:40)
at org.apache.lucene.store.DataInput.readInt(DataInput.java:86)
at org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:179)
at org.apache.lucene.index.SegmentTermEnum.<init>(SegmentTermEnum.java:51)
at org.apache.lucene.index.TermInfosReader.<init>(TermInfosReader.java:104)
at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:83)
at org.apache.lucene.index.SegmentReader.get(SegmentReader.java:116)
at org.apache.lucene.index.IndexWriter$ReaderPool.get(IndexWriter.java:696)
- locked <0x00000004007f3e80> (a org.apache.lucene.index.IndexWriter$ReaderPool)
at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4361)
at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3901)
at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:388)
at org.apache.lucene.index.TrackingConcurrentMergeScheduler.doMerge(TrackingConcurrentMergeScheduler.java:91)
at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:456)
"Attach Listener" daemon prio=10 tid=0x00000000008bb800 nid=0x6e62 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"elasticsearch[node2][generic][T#165]" daemon prio=10 tid=0x00007f5e248e2000 nid=0x6e01 waiting on condition [0x00007f5e37097000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000400063698> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#746]" daemon prio=10 tid=0x00007f5e38c1f800 nid=0x6e00 waiting on condition [0x00007f5e34877000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][http_server_worker][T#828]" daemon prio=10 tid=0x00007f5e3005e800 nid=0x6cfe waiting on condition [0x00007f5e34732000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004008ca980> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][http_server_worker][T#827]" daemon prio=10 tid=0x00007f5e30f07800 nid=0x6cfd waiting on condition [0x00007f5e35b38000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004008ca980> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#745]" daemon prio=10 tid=0x00007f5e381a5000 nid=0x6bcd waiting on condition [0x00007f5e35cbe000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][http_server_worker][T#824]" daemon prio=10 tid=0x00007f5e302df000 nid=0x682c waiting on condition [0x00007f5e34ac0000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004008ca980> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#749]" daemon prio=10 tid=0x00007f5e24495000 nid=0x682b waiting on condition [0x00007f5e1ee84000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001adf00> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#750]" daemon prio=10 tid=0x00007f5e300da800 nid=0x682a waiting on condition [0x00007f5e357ad000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001adf00> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#748]" daemon prio=10 tid=0x00007f5e31353000 nid=0x6829 waiting on condition [0x00007f5e3466f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001adf00> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#747]" daemon prio=10 tid=0x0000000000aaf800 nid=0x6827 waiting on condition [0x00007f5e34c05000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001adf00> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#746]" daemon prio=10 tid=0x0000000000c2c000 nid=0x6826 waiting on condition [0x00007f5e35e44000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001adf00> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#745]" daemon prio=10 tid=0x00007f5e30128800 nid=0x6825 runnable [0x00007f5e36466000]
java.lang.Thread.State: RUNNABLE
at org.elasticsearch.index.search.NumericRangeFieldDataFilter$4$1.get(NumericRangeFieldDataFilter.java:349)
at org.elasticsearch.common.lucene.docset.GetDocSet$1.advance(GetDocSet.java:74)
at org.apache.lucene.util.FixedBitSet.and(FixedBitSet.java:225)
at org.elasticsearch.common.lucene.docset.DocSets.and(DocSets.java:75)
at org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:121)
at org.elasticsearch.common.lucene.search.NotDeletedFilter.getDocIdSet(NotDeletedFilter.java:43)
at org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:139)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:577)
at org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:195)
at org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:149)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:487)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:400)
at org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:192)
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:234)
at org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:497)
at org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:486)
at org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:400)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#744]" daemon prio=10 tid=0x00007f5e24024800 nid=0x6824 waiting on condition [0x00007f5e34b42000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001adf00> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#743]" daemon prio=10 tid=0x00007f5e2407c800 nid=0x6822 waiting on condition [0x00007f5e362e1000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001adf00> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#742]" daemon prio=10 tid=0x00007f5e302e2000 nid=0x6821 runnable [0x00007f5e34a7f000]
java.lang.Thread.State: RUNNABLE
at org.apache.lucene.util.FixedBitSet.and(FixedBitSet.java:228)
at org.elasticsearch.common.lucene.docset.DocSets.and(DocSets.java:75)
at org.elasticsearch.common.lucene.search.XBooleanFilter.getDocIdSet(XBooleanFilter.java:121)
at org.elasticsearch.common.lucene.search.NotDeletedFilter.getDocIdSet(NotDeletedFilter.java:43)
at org.apache.lucene.search.ConstantScoreQuery$ConstantWeight.scorer(ConstantScoreQuery.java:139)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:577)
at org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:195)
at org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:149)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:487)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:400)
at org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:192)
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:234)
at org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:497)
at org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:486)
at org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:400)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][generic][T#162]" daemon prio=10 tid=0x00007f5e2a685800 nid=0x680d waiting on condition [0x00007f5e35ec6000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000400063698> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][http_server_worker][T#822]{New I/O worker #47}" daemon prio=10 tid=0x00007f5e29373000 nid=0x6787 runnable [0x00007f5e363e5000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000004632b9650> (a sun.nio.ch.Util$1)
- locked <0x00000004632b9668> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000004632b93a0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][http_server_worker][T#821]{New I/O worker #37}" daemon prio=10 tid=0x00007f5e3043a000 nid=0x6786 runnable [0x00007f5e34a3e000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000463f57820> (a sun.nio.ch.Util$1)
- locked <0x0000000463f57838> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000463f579a0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][refresh][T#479]" daemon prio=10 tid=0x00007f5e2486e000 nid=0x6745 waiting on condition [0x00007f5e362a0000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a9c30> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#741]" daemon prio=10 tid=0x00007f5e2408e000 nid=0x66f1 waiting on condition [0x00007f5e35fd5000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001adf00> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#740]" daemon prio=10 tid=0x00007f5e38f73000 nid=0x66f0 waiting on condition [0x00007f5e34773000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001adf00> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#739]" daemon prio=10 tid=0x00007f5e3ac79000 nid=0x66ef waiting on condition [0x00007f5e35d81000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001adf00> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#738]" daemon prio=10 tid=0x00007f5e316e3000 nid=0x66ee waiting on condition [0x00007f5e1eec5000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001adf00> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][http_server_worker][T#820]{New I/O worker #43}" daemon prio=10 tid=0x00007f5e30aa2000 nid=0x6363 runnable [0x00007f5e3615b000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000004632217d0> (a sun.nio.ch.Util$1)
- locked <0x00000004632217e8> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000463221690> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][refresh][T#478]" daemon prio=10 tid=0x00007f5e24081000 nid=0x635a waiting on condition [0x00007f5e35f53000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a9c30> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][refresh][T#477]" daemon prio=10 tid=0x00007f5e2408b000 nid=0x6359 waiting on condition [0x00007f5e3576c000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a9c30> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][refresh][T#476]" daemon prio=10 tid=0x00007f5e0888f800 nid=0x6283 waiting on condition [0x00007f5e364e9000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a9c30> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][refresh][T#475]" daemon prio=10 tid=0x00007f5e0849c800 nid=0x5ec9 waiting on condition [0x00007f5e357ee000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a9c30> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][http_server_worker][T#817]{New I/O worker #36}" daemon prio=10 tid=0x00007f5e3074f800 nid=0x5e72 runnable [0x00007f5e356ea000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000004633e5808> (a sun.nio.ch.Util$1)
- locked <0x00000004633e5820> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000004633e59a0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][http_server_worker][T#816]" daemon prio=10 tid=0x00007f5e316c8800 nid=0x5e71 waiting on condition [0x00007f5e35e03000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004008ca980> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][refresh][T#474]" daemon prio=10 tid=0x00007f5e0804f800 nid=0x5ddc waiting on condition [0x00007f5e3729f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a9c30> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][index][T#105]" daemon prio=10 tid=0x00007f5e085fc000 nid=0x5983 runnable [0x00007f5e346f1000]
java.lang.Thread.State: RUNNABLE
at org.elasticsearch.common.jackson.impl.Utf8Generator._writeFieldName(Utf8Generator.java:422)
at org.elasticsearch.common.jackson.impl.Utf8Generator.writeFieldName(Utf8Generator.java:282)
at org.elasticsearch.common.xcontent.json.JsonXContentGenerator.writeFieldName(JsonXContentGenerator.java:73)
at org.elasticsearch.common.xcontent.support.XContentMapConverter.writeMap(XContentMapConverter.java:120)
at org.elasticsearch.common.xcontent.support.XContentMapConverter.writeValue(XContentMapConverter.java:169)
at org.elasticsearch.common.xcontent.support.XContentMapConverter.writeIterable(XContentMapConverter.java:135)
at org.elasticsearch.common.xcontent.support.XContentMapConverter.writeValue(XContentMapConverter.java:171)
at org.elasticsearch.common.xcontent.support.XContentMapConverter.writeMap(XContentMapConverter.java:125)
at org.elasticsearch.common.xcontent.XContentBuilder.map(XContentBuilder.java:961)
at org.elasticsearch.action.index.IndexRequest.source(IndexRequest.java:372)
at org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:213)
at org.elasticsearch.action.update.TransportUpdateAction$1$1.run(TransportUpdateAction.java:236)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#743]" daemon prio=10 tid=0x00007f5e38a36000 nid=0x58dd waiting on condition [0x00007f5e36016000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#742]" daemon prio=10 tid=0x00007f5e38e31000 nid=0x58dc waiting on condition [0x00007f5e361dd000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#741]" daemon prio=10 tid=0x00007f5e085f8000 nid=0x58db waiting on condition [0x00007f5e3625f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][index][T#104]" daemon prio=10 tid=0x00007f5e24032800 nid=0x54d0 runnable [0x00007f5e366af000]
java.lang.Thread.State: RUNNABLE
at org.elasticsearch.index.mapper.core.AbstractFieldMapper.parse(AbstractFieldMapper.java:318)
at org.elasticsearch.index.mapper.object.ObjectMapper.serializeValue(ObjectMapper.java:585)
at org.elasticsearch.index.mapper.object.ObjectMapper.parse(ObjectMapper.java:449)
at org.elasticsearch.index.mapper.object.ObjectMapper.serializeObject(ObjectMapper.java:497)
at org.elasticsearch.index.mapper.object.ObjectMapper.serializeArray(ObjectMapper.java:565)
at org.elasticsearch.index.mapper.object.ObjectMapper.parse(ObjectMapper.java:441)
at org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:493)
at org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:437)
at org.elasticsearch.index.shard.service.InternalIndexShard.prepareIndex(InternalIndexShard.java:311)
at org.elasticsearch.action.index.TransportIndexAction.shardOperationOnPrimary(TransportIndexAction.java:202)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:532)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:434)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:342)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction.doExecute(TransportShardReplicationOperationAction.java:108)
at org.elasticsearch.action.index.TransportIndexAction.innerExecute(TransportIndexAction.java:135)
at org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:117)
at org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:67)
at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
at org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:220)
at org.elasticsearch.action.update.TransportUpdateAction$1$1.run(TransportUpdateAction.java:236)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#740]" daemon prio=10 tid=0x00007f5e0862e000 nid=0x54c9 waiting on condition [0x00007f5e360d9000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][refresh][T#469]" daemon prio=10 tid=0x00007f5e0849a800 nid=0x54be waiting on condition [0x00007f5e34b83000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a9c30> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#735]" daemon prio=10 tid=0x00007f5e313c2000 nid=0x5065 waiting on condition [0x00007f5e1f4da000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001adf00> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#734]" daemon prio=10 tid=0x00007f5e390d5800 nid=0x5064 waiting on condition [0x00007f5e35d40000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001adf00> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#733]" daemon prio=10 tid=0x00007f5e08607000 nid=0x5063 waiting on condition [0x00007f5e345ed000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001adf00> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#732]" daemon prio=10 tid=0x00007f5e30439800 nid=0x5062 waiting on condition [0x00007f5e356a9000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001adf00> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][search][T#730]" daemon prio=10 tid=0x00007f5e30f53000 nid=0x5060 waiting on condition [0x00007f5e35b79000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001adf00> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][index][T#102]" daemon prio=10 tid=0x00007f5e301ae000 nid=0x4b8c runnable [0x00007f5e349bb000]
java.lang.Thread.State: RUNNABLE
at org.elasticsearch.common.compress.lzf.ChunkEncoder.tryCompress(ChunkEncoder.java:236)
at org.elasticsearch.common.compress.lzf.ChunkEncoder.encodeAndWriteChunk(ChunkEncoder.java:154)
at org.elasticsearch.common.compress.lzf.LZFCompressedStreamOutput.compress(LZFCompressedStreamOutput.java:51)
at org.elasticsearch.common.compress.CompressedStreamOutput.writeBytes(CompressedStreamOutput.java:84)
at org.elasticsearch.index.mapper.internal.SourceFieldMapper.parseCreateField(SourceFieldMapper.java:285)
at org.elasticsearch.index.mapper.internal.SourceFieldMapper.parseCreateField(SourceFieldMapper.java:58)
at org.elasticsearch.index.mapper.core.AbstractFieldMapper.parse(AbstractFieldMapper.java:314)
at org.elasticsearch.index.mapper.internal.SourceFieldMapper.preParse(SourceFieldMapper.java:213)
at org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:489)
at org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:437)
at org.elasticsearch.index.shard.service.InternalIndexShard.prepareIndex(InternalIndexShard.java:311)
at org.elasticsearch.action.index.TransportIndexAction.shardOperationOnPrimary(TransportIndexAction.java:202)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:532)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:434)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:342)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction.doExecute(TransportShardReplicationOperationAction.java:108)
at org.elasticsearch.action.index.TransportIndexAction.innerExecute(TransportIndexAction.java:135)
at org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:117)
at org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:67)
at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
at org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:220)
at org.elasticsearch.action.update.TransportUpdateAction$1$1.run(TransportUpdateAction.java:236)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][index][T#101]" daemon prio=10 tid=0x00007f5e08864000 nid=0x4b8b runnable [0x00007f5e35e84000]
java.lang.Thread.State: RUNNABLE
at org.elasticsearch.common.compress.lzf.ChunkEncoder.tryCompress(ChunkEncoder.java:236)
at org.elasticsearch.common.compress.lzf.ChunkEncoder.encodeAndWriteChunk(ChunkEncoder.java:154)
at org.elasticsearch.common.compress.lzf.LZFCompressedStreamOutput.compress(LZFCompressedStreamOutput.java:51)
at org.elasticsearch.common.compress.CompressedStreamOutput.flushBuffer(CompressedStreamOutput.java:122)
at org.elasticsearch.common.compress.CompressedStreamOutput.close(CompressedStreamOutput.java:105)
at org.elasticsearch.index.mapper.internal.SourceFieldMapper.parseCreateField(SourceFieldMapper.java:286)
at org.elasticsearch.index.mapper.internal.SourceFieldMapper.parseCreateField(SourceFieldMapper.java:58)
at org.elasticsearch.index.mapper.core.AbstractFieldMapper.parse(AbstractFieldMapper.java:314)
at org.elasticsearch.index.mapper.internal.SourceFieldMapper.preParse(SourceFieldMapper.java:213)
at org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:489)
at org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:437)
at org.elasticsearch.index.shard.service.InternalIndexShard.prepareIndex(InternalIndexShard.java:311)
at org.elasticsearch.action.index.TransportIndexAction.shardOperationOnPrimary(TransportIndexAction.java:202)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:532)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:434)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.start(TransportShardReplicationOperationAction.java:342)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction.doExecute(TransportShardReplicationOperationAction.java:108)
at org.elasticsearch.action.index.TransportIndexAction.innerExecute(TransportIndexAction.java:135)
at org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:117)
at org.elasticsearch.action.index.TransportIndexAction.doExecute(TransportIndexAction.java:67)
at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
at org.elasticsearch.action.update.TransportUpdateAction.shardOperation(TransportUpdateAction.java:220)
at org.elasticsearch.action.update.TransportUpdateAction$1$1.run(TransportUpdateAction.java:236)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][index][T#100]" daemon prio=10 tid=0x00007f5e24095000 nid=0x4b8a waiting on condition [0x00007f5e36322000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001b4518> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#738]" daemon prio=10 tid=0x00007f5e30eb6800 nid=0x4abc waiting on condition [0x00007f5e35c3c000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#737]" daemon prio=10 tid=0x00007f5e300f8800 nid=0x4abb waiting on condition [0x00007f5e34d8b000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#735]" daemon prio=10 tid=0x00007f5e30810000 nid=0x4ab9 waiting on condition [0x00007f5e3582f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#733]" daemon prio=10 tid=0x00007f5e30413000 nid=0x4ab7 waiting on condition [0x00007f5e363a4000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#732]" daemon prio=10 tid=0x00007f5e080b4000 nid=0x4ab6 waiting on condition [0x00007f5e35f07000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#730]" daemon prio=10 tid=0x00007f5e08499800 nid=0x4ab4 waiting on condition [0x00007f5e35dc2000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#729]" daemon prio=10 tid=0x00007f5e308dc800 nid=0x4659 waiting on condition [0x00007f5e345ac000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][refresh][T#461]" daemon prio=10 tid=0x00007f5e0888e000 nid=0x3c10 waiting on condition [0x00007f5e35f94000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a9c30> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#726]" daemon prio=10 tid=0x00007f5e30724000 nid=0x3b7a waiting on condition [0x00007f5e35bfb000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#725]" daemon prio=10 tid=0x00007f5e399cb000 nid=0x3b77 waiting on condition [0x00007f5e35cff000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#721]" daemon prio=10 tid=0x00007f5e3930c000 nid=0x360e waiting on condition [0x00007f5e3666f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#719]" daemon prio=10 tid=0x00007f5e08897800 nid=0x2d04 waiting on condition [0x00007f5e35c7d000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][index][T#92]" daemon prio=10 tid=0x000000000153d000 nid=0x39c waiting on condition [0x00007f5e36363000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001b4518> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#700]" daemon prio=10 tid=0x00007f5e38d9d800 nid=0x6d65 waiting on condition [0x00007f5e36b01000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#692]" daemon prio=10 tid=0x00007f5e302f9000 nid=0x5fe5 waiting on condition [0x00007f5e3619c000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#688]" daemon prio=10 tid=0x00007f5e30062000 nid=0x51ff waiting on condition [0x00007f5e3497b000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][bulk][T#676]" daemon prio=10 tid=0x00007f5e307db800 nid=0x4085 waiting on condition [0x00007f5e35668000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001af918> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:453)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:352)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][flush][T#17]" daemon prio=10 tid=0x00007f5e3930b000 nid=0x5eab waiting on condition [0x00007f5e349fd000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001acbb0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:706)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.take(LinkedTransferQueue.java:1109)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][merge][T#8]" daemon prio=10 tid=0x0000000000acf800 nid=0x3a62 waiting on condition [0x00007f5e358b1000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a6500> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][merge][T#7]" daemon prio=10 tid=0x00000000008a7000 nid=0x3a61 waiting on condition [0x00007f5e36057000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a6500> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][merge][T#6]" daemon prio=10 tid=0x0000000000b6f800 nid=0x3a1a waiting on condition [0x00007f5e1ef06000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a6500> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][merge][T#5]" daemon prio=10 tid=0x0000000002932000 nid=0x3a10 waiting on condition [0x00007f5e3611a000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a6500> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][merge][T#4]" daemon prio=10 tid=0x000000000091e800 nid=0x2b73 waiting on condition [0x00007f5e34b01000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a6500> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][merge][T#3]" daemon prio=10 tid=0x0000000000bee000 nid=0x2730 waiting on condition [0x00007f5e3462e000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a6500> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][snapshot][T#5]" daemon prio=10 tid=0x0000000000c6d000 nid=0x2589 waiting on condition [0x00007f5e36b83000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a63e0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][snapshot][T#4]" daemon prio=10 tid=0x0000000001745800 nid=0x2258 waiting on condition [0x00007f5e346b0000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a63e0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][snapshot][T#3]" daemon prio=10 tid=0x0000000001119800 nid=0x1bf7 waiting on condition [0x00007f5e347b4000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a63e0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][management][T#5]" daemon prio=10 tid=0x0000000001100000 nid=0x1bf3 waiting on condition [0x00007f5e347f5000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000040008beb0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][merge][T#2]" daemon prio=10 tid=0x000000000125f000 nid=0x1bf2 waiting on condition [0x00007f5e34836000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a6500> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][snapshot][T#2]" daemon prio=10 tid=0x0000000001276000 nid=0x166c waiting on condition [0x00007f5e3715a000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a63e0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][management][T#4]" daemon prio=10 tid=0x00007f5e38bca000 nid=0x131b waiting on condition [0x00007f5e34cc8000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000040008beb0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][http_server_worker][T#2]{New I/O worker #41}" daemon prio=10 tid=0x000000000109e800 nid=0x130b runnable [0x00007f5e36426000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000408a3f160> (a sun.nio.ch.Util$1)
- locked <0x0000000408a3f178> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000408a404a0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][snapshot][T#1]" daemon prio=10 tid=0x0000000000b73000 nid=0x12c3 waiting on condition [0x00007f5e3652a000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a63e0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][merge][T#1]" daemon prio=10 tid=0x00000000009b6000 nid=0x12c2 waiting on condition [0x00007f5e3656b000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a6500> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][management][T#3]" daemon prio=10 tid=0x000000000089a800 nid=0x12c0 waiting on condition [0x00007f5e365ed000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000040008beb0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][management][T#2]" daemon prio=10 tid=0x00007f5e304ca800 nid=0x12bc waiting on condition [0x00007f5e366f1000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000040008beb0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][cache][T#4]" daemon prio=10 tid=0x0000000000a1a000 nid=0x12b8 waiting on condition [0x00007f5e36732000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a6440> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][cache][T#3]" daemon prio=10 tid=0x0000000000a17800 nid=0x12b7 runnable [0x00007f5e36773000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.NativeThread.current(Native Method)
at sun.nio.ch.NativeThreadSet.add(NativeThreadSet.java:45)
at sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:661)
at org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:162)
at org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:270)
at org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:136)
at org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:111)
at org.apache.lucene.index.TermBuffer.read(TermBuffer.java:82)
at org.apache.lucene.index.SegmentTermEnum.next(SegmentTermEnum.java:133)
at org.elasticsearch.index.cache.bloom.simple.SimpleBloomCache$BloomFilterLoader.run(SimpleBloomCache.java:203)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][cache][T#2]" daemon prio=10 tid=0x0000000000a15800 nid=0x12b6 waiting on condition [0x00007f5e367b4000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a6440> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][cache][T#1]" daemon prio=10 tid=0x0000000000c88800 nid=0x12b5 waiting on condition [0x00007f5e367f5000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004001a6440> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][management][T#1]" daemon prio=10 tid=0x00007f5e302fc800 nid=0x11da waiting on condition [0x00007f5e36877000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000040008beb0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:702)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.xfer(LinkedTransferQueue.java:615)
at org.elasticsearch.common.util.concurrent.jsr166y.LinkedTransferQueue.poll(LinkedTransferQueue.java:1117)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"DestroyJavaVM" prio=10 tid=0x00007f5e38221000 nid=0xffc waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"elasticsearch[keepAlive]" prio=10 tid=0x00007f5e3821f800 nid=0x11d9 waiting on condition [0x00007f5e368b8000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000040008bb50> (a java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:838)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:998)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:235)
at org.elasticsearch.bootstrap.Bootstrap$3.run(Bootstrap.java:222)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][http_server_boss][T#1]{New I/O server boss #2 ([id: 0x1c7b0f4d, /0.0.0.0:9200])}" daemon prio=10 tid=0x00007f5e386c9000 nid=0x11d8 runnable [0x00007f5e368f9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400061438> (a sun.nio.ch.Util$1)
- locked <0x0000000400061450> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400060fb8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.NioServerSocketPipelineSink$Boss.run(NioServerSocketPipelineSink.java:227)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_client_worker][T#16]{New I/O worker #13}" daemon prio=10 tid=0x00007f5e386c6800 nid=0x11d7 runnable [0x00007f5e3693a000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400063638> (a sun.nio.ch.Util$1)
- locked <0x0000000400063650> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400062708> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][clusterService#updateTask][T#1]" daemon prio=10 tid=0x0000000000c1d800 nid=0x11d6 waiting on condition [0x00007f5e3697b000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000004008c31f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:386)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_server_worker][T#16]{New I/O worker #25}" daemon prio=10 tid=0x00007f5e3005b000 nid=0x11d5 runnable [0x00007f5e369bc000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000004000614b0> (a sun.nio.ch.Util$1)
- locked <0x00000004000614c8> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400061048> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_server_worker][T#15]{New I/O worker #24}" daemon prio=10 tid=0x00007f5e30059000 nid=0x11d4 runnable [0x00007f5e369fd000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400061510> (a sun.nio.ch.Util$1)
- locked <0x0000000400061528> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400061120> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_server_worker][T#14]{New I/O worker #23}" daemon prio=10 tid=0x00007f5e30057000 nid=0x11d3 runnable [0x00007f5e36a3e000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400061570> (a sun.nio.ch.Util$1)
- locked <0x0000000400061588> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000004000611b0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_server_worker][T#13]{New I/O worker #22}" daemon prio=10 tid=0x00007f5e30111800 nid=0x11d2 runnable [0x00007f5e36a7f000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000004000625a0> (a sun.nio.ch.Util$1)
- locked <0x00000004000625b8> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400061240> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_server_worker][T#12]{New I/O worker #21}" daemon prio=10 tid=0x00007f5e30110800 nid=0x11d1 runnable [0x00007f5e371dc000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400062600> (a sun.nio.ch.Util$1)
- locked <0x0000000400062618> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000004000612d0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_server_worker][T#11]{New I/O worker #20}" daemon prio=10 tid=0x00007f5e30110000 nid=0x11d0 runnable [0x00007f5e3721d000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400062660> (a sun.nio.ch.Util$1)
- locked <0x0000000400062678> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400061360> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_server_worker][T#10]{New I/O worker #19}" daemon prio=10 tid=0x00007f5e3010f000 nid=0x11cf runnable [0x00007f5e3725e000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400065030> (a sun.nio.ch.Util$1)
- locked <0x0000000400065018> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400089270> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_client_worker][T#15]{New I/O worker #2}" daemon prio=10 tid=0x00007f5e386c4800 nid=0x11ce runnable [0x00007f5e36ac0000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400062c60> (a sun.nio.ch.Util$1)
- locked <0x0000000400062c78> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400065078> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_client_worker][T#14]{New I/O worker #3}" daemon prio=10 tid=0x00007f5e38364800 nid=0x11cc runnable [0x00007f5e36b42000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400065ab8> (a sun.nio.ch.Util$1)
- locked <0x0000000400065ad0> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400064868> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_client_worker][T#13]{New I/O worker #16}" daemon prio=10 tid=0x00007f5e38730800 nid=0x11ca runnable [0x00007f5e36bc4000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400065b18> (a sun.nio.ch.Util$1)
- locked <0x0000000400065b30> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000004000648f8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_client_worker][T#12]{New I/O worker #14}" daemon prio=10 tid=0x00007f5e38339000 nid=0x11c9 runnable [0x00007f5e36c05000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400065b78> (a sun.nio.ch.Util$1)
- locked <0x0000000400065b90> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400064988> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_client_worker][T#11]{New I/O worker #12}" daemon prio=10 tid=0x00007f5e38338000 nid=0x11c8 runnable [0x00007f5e36c46000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400065bd8> (a sun.nio.ch.Util$1)
- locked <0x0000000400065bf0> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400064a18> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_client_worker][T#10]{New I/O worker #5}" daemon prio=10 tid=0x00007f5e38416800 nid=0x11c7 runnable [0x00007f5e36c87000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400065dd8> (a sun.nio.ch.Util$1)
- locked <0x0000000400065df0> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400064aa8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_client_worker][T#9]{New I/O worker #8}" daemon prio=10 tid=0x00007f5e3838c800 nid=0x11c6 runnable [0x00007f5e36cc8000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400065e38> (a sun.nio.ch.Util$1)
- locked <0x0000000400065e50> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400064b38> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_client_worker][T#8]{New I/O worker #10}" daemon prio=10 tid=0x00007f5e3828b800 nid=0x11c5 runnable [0x00007f5e36d09000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400065f40> (a sun.nio.ch.Util$1)
- locked <0x0000000400065f28> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400064bc8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_server_worker][T#9]{New I/O worker #28}" daemon prio=10 tid=0x00007f5e3005c000 nid=0x11c4 runnable [0x00007f5e36d4a000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000004000636c8> (a sun.nio.ch.Util$1)
- locked <0x00000004000636e0> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400062828> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_client_worker][T#7]{New I/O worker #9}" daemon prio=10 tid=0x00007f5e383e7800 nid=0x11c3 runnable [0x00007f5e36d8b000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400061a10> (a sun.nio.ch.Util$1)
- locked <0x0000000400061a28> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400089468> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_server_worker][T#8]{New I/O worker #27}" daemon prio=10 tid=0x00007f5e30051800 nid=0x11c2 runnable [0x00007f5e36dcc000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400063728> (a sun.nio.ch.Util$1)
- locked <0x0000000400063740> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000004000628b8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_client_worker][T#6]{New I/O worker #6}" daemon prio=10 tid=0x00007f5e3833c000 nid=0x11c1 runnable [0x00007f5e36e0d000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400063788> (a sun.nio.ch.Util$1)
- locked <0x00000004000637a0> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400062948> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_server_worker][T#7]{New I/O worker #29}" daemon prio=10 tid=0x00007f5e3004f800 nid=0x11c0 runnable [0x00007f5e36e4e000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400061a70> (a sun.nio.ch.Util$1)
- locked <0x0000000400061a88> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000004000894f8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_client_worker][T#5]{New I/O worker #11}" daemon prio=10 tid=0x00007f5e38411000 nid=0x11bf runnable [0x00007f5e36e8f000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400061ad0> (a sun.nio.ch.Util$1)
- locked <0x0000000400061ae8> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400089588> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_server_worker][T#6]{New I/O worker #30}" daemon prio=10 tid=0x00007f5e30068800 nid=0x11be runnable [0x00007f5e36ed0000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400089c50> (a sun.nio.ch.Util$1)
- locked <0x0000000400089c68> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000004000629d8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_client_worker][T#4]{New I/O worker #7}" daemon prio=10 tid=0x00007f5e38280800 nid=0x11bd runnable [0x00007f5e36f11000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x000000040086eb10> (a sun.nio.ch.Util$1)
- locked <0x000000040086eaf8> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400089660> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_server_worker][T#5]{New I/O worker #26}" daemon prio=10 tid=0x00007f5e30035000 nid=0x11bc runnable [0x00007f5e36f52000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400089cb0> (a sun.nio.ch.Util$1)
- locked <0x0000000400089cc8> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400062a68> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_client_worker][T#3]{New I/O worker #4}" daemon prio=10 tid=0x00007f5e38350800 nid=0x11bb runnable [0x00007f5e36f93000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400089d10> (a sun.nio.ch.Util$1)
- locked <0x0000000400089d28> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400062af8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_server_worker][T#4]{New I/O worker #32}" daemon prio=10 tid=0x00007f5e3001a000 nid=0x11ba runnable [0x00007f5e36fd4000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x000000040008c220> (a sun.nio.ch.Util$1)
- locked <0x000000040008c238> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000400093ab8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_client_worker][T#2]{New I/O worker #15}" daemon prio=10 tid=0x0000000000c6b000 nid=0x11b9 runnable [0x00007f5e37015000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x000000040008eb18> (a sun.nio.ch.Util$1)
- locked <0x000000040008eb30> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000040008e6e0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_server_worker][T#3]{New I/O worker #31}" daemon prio=10 tid=0x0000000000c69000 nid=0x11b8 runnable [0x00007f5e37056000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x000000040008eb78> (a sun.nio.ch.Util$1)
- locked <0x000000040008eb90> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000040008e770> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_client_worker][T#1]{New I/O worker #1}" daemon prio=10 tid=0x0000000000db8000 nid=0x11b6 runnable [0x00007f5e370d8000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x000000040008ebd8> (a sun.nio.ch.Util$1)
- locked <0x000000040008ebf0> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000040008e800> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_server_worker][T#2]{New I/O worker #17}" daemon prio=10 tid=0x0000000000b33800 nid=0x11b5 runnable [0x00007f5e37119000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x000000040008ec38> (a sun.nio.ch.Util$1)
- locked <0x000000040008ec50> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000040008e890> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_server_worker][T#1]{New I/O worker #18}" daemon prio=10 tid=0x00007f5e3837d000 nid=0x11b3 runnable [0x00007f5e3719b000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x0000000400090b50> (a sun.nio.ch.Util$1)
- locked <0x0000000400090b38> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000004000896f0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:209)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][transport_server_boss][T#1]{New I/O server boss #1 ([id: 0x03b4d679, /0.0.0.0:9300])}" daemon prio=10 tid=0x00007f5e3846b000 nid=0x11ae runnable [0x00007f5e372e0000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:83)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x000000040008f960> (a sun.nio.ch.Util$1)
- locked <0x000000040008f948> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000040008e920> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.elasticsearch.common.netty.channel.socket.nio.NioServerSocketPipelineSink$Boss.run(NioServerSocketPipelineSink.java:227)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][[ttl_expire]]" daemon prio=10 tid=0x00007f5e38444000 nid=0x11ad waiting on condition [0x00007f5e3741f000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at org.elasticsearch.indices.ttl.IndicesTTLService$PurgerThread.run(IndicesTTLService.java:135)
"elasticsearch[node2][scheduler][T#1]" daemon prio=10 tid=0x00007f5e38341000 nid=0x11ac waiting on condition [0x00007f5e37460000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000040008c280> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2081)
at java.util.concurrent.DelayQueue.take(DelayQueue.java:193)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:688)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:681)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"elasticsearch[node2][[timer]]" daemon prio=10 tid=0x00007f5e38251000 nid=0x11ab waiting on condition [0x00007f5e374a1000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at org.elasticsearch.threadpool.ThreadPool$EstimatedTimeThread.run(ThreadPool.java:365)
"RMI TCP Accept-0" daemon prio=10 tid=0x0000000000b4f800 nid=0x1149 runnable [0x00007f5e37a39000]
java.lang.Thread.State: RUNNABLE
at java.net.PlainSocketImpl.socketAccept(Native Method)
at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:375)
at java.net.ServerSocket.implAccept(ServerSocket.java:470)
at java.net.ServerSocket.accept(ServerSocket.java:438)
at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52)
at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:387)
at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:359)
at java.lang.Thread.run(Thread.java:679)
"spm-monitor-1-thread" prio=10 tid=0x00007f5e381ec000 nid=0x111f waiting on condition [0x00007f5e37a7a000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000040008c2e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2081)
at java.util.concurrent.DelayQueue.take(DelayQueue.java:193)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:688)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:681)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
"Low Memory Detector" daemon prio=10 tid=0x00007f5e3819b800 nid=0x1089 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread1" daemon prio=10 tid=0x00007f5e38190800 nid=0x1088 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" daemon prio=10 tid=0x00007f5e3818a800 nid=0x1087 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"spm-bootstrap-thread" prio=10 tid=0x00007f5e38181800 nid=0x1086 waiting on condition [0x00007f5e37cfe000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000040008c340> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2081)
at java.util.concurrent.ThreadPoolExecutor.awaitTermination(ThreadPoolExecutor.java:1425)
at java.util.concurrent.Executors$DelegatedExecutorService.awaitTermination(Executors.java:636)
at com.sematext.spm.MonitorAgent$1.run(MonitorAgent.java:115)
at java.lang.Thread.run(Thread.java:679)
"Signal Dispatcher" daemon prio=10 tid=0x0000000000839800 nid=0x1010 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Surrogate Locker Thread (Concurrent GC)" daemon prio=10 tid=0x0000000000838800 nid=0x100f waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" daemon prio=10 tid=0x000000000081a000 nid=0x100d in Object.wait() [0x00007f5e5c7e1000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x0000000400095138> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:133)
- locked <0x0000000400095138> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:149)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:177)
"Reference Handler" daemon prio=10 tid=0x0000000000812800 nid=0x100b in Object.wait() [0x00007f5e3dafd000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x0000000400061b30> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:502)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133)
- locked <0x0000000400061b30> (a java.lang.ref.Reference$Lock)
"VM Thread" prio=10 tid=0x000000000080b800 nid=0x1009 runnable
"Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x000000000061b800 nid=0xffd runnable
"Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x000000000061d800 nid=0xffe runnable
"Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x000000000061f800 nid=0xfff runnable
"Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x0000000000621000 nid=0x1000 runnable
"Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x0000000000623000 nid=0x1001 runnable
"Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x0000000000625000 nid=0x1002 runnable
"Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x0000000000626800 nid=0x1003 runnable
"Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x0000000000628800 nid=0x1004 runnable
"Concurrent Mark-Sweep GC Thread" prio=10 tid=0x000000000071c800 nid=0x1008 runnable
"Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x0000000000718000 nid=0x1006 runnable
"Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x000000000071a000 nid=0x1007 runnable
"VM Periodic Task Thread" prio=10 tid=0x0000000000a1c000 nid=0x11aa waiting on condition
JNI global references: 1452
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment