Skip to content

Instantly share code, notes, and snippets.

@Avatah
Created August 9, 2011 10:43
Show Gist options
  • Save Avatah/1133740 to your computer and use it in GitHub Desktop.
Save Avatah/1133740 to your computer and use it in GitHub Desktop.
Deadlock in ES 0.17.4
2011-08-09 12:39:17
Full thread dump Java HotSpot(TM) 64-Bit Server VM (16.2-b04 mixed mode):
"Attach Listener" daemon prio=10 tid=0x00007f5468007000 nid=0x5125 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"elasticsearch[merge]-pool-184-thread-1" daemon prio=10 tid=0x00007f5464c6c000 nid=0x50c3 waiting on condition [0x00007f546cc38000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f54c4783b78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"elasticsearch[index]-pool-180-thread-1" daemon prio=10 tid=0x00007f54dd1fa000 nid=0x50c1 waiting for monitor entry [0x00007f546c934000]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.elasticsearch.index.mapper.MapperService$InternalFieldMapperListener.fieldMapper(MapperService.java:710)
- waiting to lock <0x00007f54c5133670> (a java.lang.Object)
at org.elasticsearch.index.mapper.DocumentMapper.addFieldMapper(DocumentMapper.java:634)
- locked <0x00007f54c620bf28> (a java.lang.Object)
at org.elasticsearch.index.mapper.object.ObjectMapper$3.fieldMapper(ObjectMapper.java:695)
at org.elasticsearch.index.mapper.core.AbstractFieldMapper.traverse(AbstractFieldMapper.java:323)
at org.elasticsearch.index.mapper.object.ObjectMapper.serializeValue(ObjectMapper.java:693)
- locked <0x00007f54c620b9c0> (a java.lang.Object)
at org.elasticsearch.index.mapper.object.ObjectMapper.parse(ObjectMapper.java:440)
at org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:566)
at org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:490)
at org.elasticsearch.index.shard.service.InternalIndexShard.prepareIndex(InternalIndexShard.java:289)
at org.elasticsearch.action.index.TransportIndexAction.shardOperationOnPrimary(TransportIndexAction.java:185)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:428)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$1.run(TransportShardReplicationOperationAction.java:341)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
"elasticsearch[management]-pool-183-thread-1" daemon prio=10 tid=0x00007f54dd1d2000 nid=0x50bf waiting on condition [0x00007f546ce3a000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f54c47822b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"New I/O server boss #5 ([id: 0x65b57dcc, /0:0:0:0:0:0:0:0:9203])" daemon prio=10 tid=0x00007f54dd11d000 nid=0x50bd runnable [0x00007f546cd39000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
- locked <0x00007f54c4a6e290> (a sun.nio.ch.Util$1)
- locked <0x00007f54c4a6e278> (a java.util.Collections$UnmodifiableSet)
- locked <0x00007f54c4a6df18> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at org.elasticsearch.common.netty.channel.socket.nio.NioServerSocketPipelineSink$Boss.run(NioServerSocketPipelineSink.java:240)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:44)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
"elasticsearch[Rhiannon]riverClusterService#updateTask-pool-188-thread-1" daemon prio=10 tid=0x00007f5478003800 nid=0x50bc waiting on condition [0x00007f546cb37000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f54c4a42900> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"elasticsearch[cached]-pool-179-thread-1" daemon prio=10 tid=0x00007f5478007000 nid=0x50bb waiting on condition [0x00007f546c32f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f54c477db10> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:424)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:323)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:874)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:945)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"elasticsearch[Rhiannon]clusterService#updateTask-pool-189-thread-1" daemon prio=10 tid=0x00007f54dcd27000 nid=0x50ba waiting for monitor entry [0x00007f546c430000]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.elasticsearch.index.mapper.DocumentMapper.addObjectMapperListener(DocumentMapper.java:670)
- waiting to lock <0x00007f54c620bf28> (a java.lang.Object)
at org.elasticsearch.index.mapper.MapperService.add(MapperService.java:187)
- locked <0x00007f54c5133670> (a java.lang.Object)
at org.elasticsearch.index.mapper.MapperService.add(MapperService.java:166)
at org.elasticsearch.indices.cluster.IndicesClusterStateService.processMapping(IndicesClusterStateService.java:382)
at org.elasticsearch.indices.cluster.IndicesClusterStateService.applyMappings(IndicesClusterStateService.java:349)
at org.elasticsearch.indices.cluster.IndicesClusterStateService.clusterChanged(IndicesClusterStateService.java:176)
- locked <0x00007f54c4a1f168> (a java.lang.Object)
at org.elasticsearch.cluster.service.InternalClusterService$2.run(InternalClusterService.java:254)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
"elasticsearch[Rhiannon][scheduler]-pool-186-thread-1" daemon prio=10 tid=0x00007f54dcf3c000 nid=0x50b9 waiting on condition [0x00007f546c733000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f54c4785e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
at java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"elasticsearch[Rhiannon][timer]" daemon prio=10 tid=0x00007f54dcbd1000 nid=0x50b8 waiting on condition [0x00007f546c834000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at org.elasticsearch.threadpool.ThreadPool$EstimatedTimeThread.run(ThreadPool.java:282)
"ForkJoinScheduler" daemon prio=10 tid=0x00007f54dd066800 nid=0x507b in Object.wait() [0x00007f546cf3b000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at scala.actors.scheduler.ForkJoinScheduler.liftedTree1$1(ForkJoinScheduler.scala:70)
at scala.actors.scheduler.ForkJoinScheduler.run(ForkJoinScheduler.scala:69)
- locked <0x00007f549a7f53d8> (a scala.actors.scheduler.ForkJoinScheduler)
at java.lang.Thread.run(Thread.java:619)
"ForkJoinPool-2-worker-3" daemon prio=10 tid=0x00007f54dcd32000 nid=0x507a waiting on condition [0x00007f546d03c000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f54c54c0418> (a scala.concurrent.forkjoin.ForkJoinPool$WaitQueueNode)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at scala.concurrent.forkjoin.ForkJoinPool$WaitQueueNode.awaitSyncRelease(ForkJoinPool.java:1340)
at scala.concurrent.forkjoin.ForkJoinPool.sync(ForkJoinPool.java:1417)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.mainLoop(ForkJoinWorkerThread.java:342)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:325)
"ForkJoinPool-2-worker-2" daemon prio=10 tid=0x00007f54dd104800 nid=0x5079 waiting on condition [0x00007f546d13d000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f54c54c0440> (a scala.concurrent.forkjoin.ForkJoinPool$WaitQueueNode)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at scala.concurrent.forkjoin.ForkJoinPool$WaitQueueNode.awaitSyncRelease(ForkJoinPool.java:1340)
at scala.concurrent.forkjoin.ForkJoinPool.sync(ForkJoinPool.java:1417)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.mainLoop(ForkJoinWorkerThread.java:342)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:325)
"ForkJoinPool-2-worker-1" daemon prio=10 tid=0x00007f54dd136800 nid=0x5078 waiting on condition [0x00007f546d23e000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f54c54c02e8> (a scala.concurrent.forkjoin.ForkJoinPool$WaitQueueNode)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at scala.concurrent.forkjoin.ForkJoinPool$WaitQueueNode.awaitSyncRelease(ForkJoinPool.java:1340)
at scala.concurrent.forkjoin.ForkJoinPool.sync(ForkJoinPool.java:1417)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.mainLoop(ForkJoinWorkerThread.java:342)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:325)
"ForkJoinPool-2-worker-0" daemon prio=10 tid=0x00007f54dc23a000 nid=0x5077 waiting on condition [0x00007f546d33f000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f54c54c03f0> (a scala.concurrent.forkjoin.ForkJoinPool$WaitQueueNode)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at scala.concurrent.forkjoin.ForkJoinPool$WaitQueueNode.awaitSyncRelease(ForkJoinPool.java:1340)
at scala.concurrent.forkjoin.ForkJoinPool.sync(ForkJoinPool.java:1417)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.mainLoop(ForkJoinWorkerThread.java:342)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:325)
"elasticsearch[management]-pool-125-thread-1" daemon prio=10 tid=0x00007f54dd148000 nid=0x5074 waiting on condition [0x00007f546d440000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f549a5d47f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"elasticsearch[cached]-pool-121-thread-2" daemon prio=10 tid=0x00007f5458001800 nid=0x506f waiting on condition [0x00007f546d54b000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f549a5d6450> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:424)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:323)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:874)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:945)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"New I/O server boss #1 ([id: 0x3865a686, /0:0:0:0:0:0:0:0:9202])" daemon prio=10 tid=0x00007f54dcf7a800 nid=0x506e runnable [0x00007f546d64c000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
- locked <0x00007f549a5b0500> (a sun.nio.ch.Util$1)
- locked <0x00007f549a5b04e8> (a java.util.Collections$UnmodifiableSet)
- locked <0x00007f549a5b0128> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at org.elasticsearch.common.netty.channel.socket.nio.NioServerSocketPipelineSink$Boss.run(NioServerSocketPipelineSink.java:240)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:44)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
"elasticsearch[Paris]riverClusterService#updateTask-pool-130-thread-1" daemon prio=10 tid=0x00007f545c003800 nid=0x506d waiting on condition [0x00007f546d74d000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f549a7207c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"elasticsearch[Paris]clusterService#updateTask-pool-131-thread-1" daemon prio=10 tid=0x00007f54dd016000 nid=0x506b waiting on condition [0x00007f546d94f000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f549a5b1bc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"elasticsearch[Paris][scheduler]-pool-128-thread-1" daemon prio=10 tid=0x00007f54dd011000 nid=0x5069 waiting on condition [0x00007f546da50000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f549a5f2730> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
at java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"org.elasticsearch.common.base.internal.Finalizer" daemon prio=10 tid=0x00007f54dcbe4800 nid=0x5068 in Object.wait() [0x00007f546db51000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00007f549a8ed950> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
- locked <0x00007f549a8ed950> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134)
at org.elasticsearch.common.base.internal.Finalizer.run(Finalizer.java:127)
"elasticsearch[Paris][timer]" daemon prio=10 tid=0x00007f54dcf29000 nid=0x5067 waiting on condition [0x00007f546dc52000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at org.elasticsearch.threadpool.ThreadPool$EstimatedTimeThread.run(ThreadPool.java:282)
"ForkJoinScheduler" prio=10 tid=0x00007f54dcbd8800 nid=0x5063 in Object.wait() [0x00007f546e0d3000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at scala.actors.scheduler.ForkJoinScheduler.liftedTree1$1(ForkJoinScheduler.scala:70)
at scala.actors.scheduler.ForkJoinScheduler.run(ForkJoinScheduler.scala:69)
- locked <0x00007f549a31fb50> (a scala.actors.scheduler.ForkJoinScheduler)
at java.lang.Thread.run(Thread.java:619)
"ForkJoinPool-1-worker-3" daemon prio=10 tid=0x00007f54dcb43800 nid=0x5062 waiting on condition [0x00007f54881cd000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f549a325dc0> (a scala.concurrent.forkjoin.ForkJoinPool$WaitQueueNode)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at scala.concurrent.forkjoin.ForkJoinPool$WaitQueueNode.awaitSyncRelease(ForkJoinPool.java:1340)
at scala.concurrent.forkjoin.ForkJoinPool.sync(ForkJoinPool.java:1417)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.mainLoop(ForkJoinWorkerThread.java:342)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:325)
"ForkJoinPool-1-worker-2" daemon prio=10 tid=0x00007f54dcbf3800 nid=0x5061 waiting on condition [0x00007f54884d0000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f549a325de8> (a scala.concurrent.forkjoin.ForkJoinPool$WaitQueueNode)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at scala.concurrent.forkjoin.ForkJoinPool$WaitQueueNode.awaitSyncRelease(ForkJoinPool.java:1340)
at scala.concurrent.forkjoin.ForkJoinPool.sync(ForkJoinPool.java:1417)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.mainLoop(ForkJoinWorkerThread.java:342)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:325)
"ForkJoinPool-1-worker-1" daemon prio=10 tid=0x00007f54dcc2f000 nid=0x5060 in Object.wait() [0x00007f5488bf9000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00007f549a3263c0> (a scala.actors.Actor$$anon$1)
at java.lang.Object.wait(Object.java:485)
at scala.actors.Actor$class.liftedTree1$1(Actor.scala:644)
at scala.actors.Actor$class.scala$actors$Actor$$suspendActor(Actor.scala:643)
- locked <0x00007f549a3263c0> (a scala.actors.Actor$$anon$1)
at scala.actors.Actor$blocker$.block(Actor.scala:634)
at scala.actors.scheduler.ForkJoinScheduler$$anon$2.block(ForkJoinScheduler.scala:145)
at scala.concurrent.forkjoin.ForkJoinPool.awaitBlocker(ForkJoinPool.java:1791)
at scala.concurrent.forkjoin.ForkJoinPool.managedBlock(ForkJoinPool.java:1781)
at scala.actors.scheduler.ForkJoinScheduler.managedBlock(ForkJoinScheduler.scala:144)
at scala.actors.scheduler.DelegatingScheduler$class.managedBlock(DelegatingScheduler.scala:73)
at scala.actors.Scheduler$.managedBlock(Scheduler.scala:21)
at scala.actors.Actor$class.receive(Actor.scala:512)
- locked <0x00007f549a3263c0> (a scala.actors.Actor$$anon$1)
at scala.actors.Actor$$anon$1.receive(Actor.scala:134)
at scala.actors.Actor$.receive(Actor.scala:189)
at org.scalatest.DispatchReporter$$anonfun$1.apply(DispatchReporter.scala:101)
at org.scalatest.DispatchReporter$$anonfun$1.apply(DispatchReporter.scala:50)
at scala.Function0$class.apply$mcV$sp(Function0.scala:39)
at scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:17)
at scala.actors.Actor$$anon$1.act(Actor.scala:135)
at scala.actors.Reactor$$anonfun$dostart$1.apply(Reactor.scala:222)
at scala.actors.Reactor$$anonfun$dostart$1.apply(Reactor.scala:222)
at scala.actors.ReactorTask.run(ReactorTask.scala:36)
at scala.concurrent.forkjoin.ForkJoinPool$AdaptedRunnable.exec(ForkJoinPool.java:611)
at scala.concurrent.forkjoin.ForkJoinTask.quietlyExec(ForkJoinTask.java:422)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.mainLoop(ForkJoinWorkerThread.java:340)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:325)
"ForkJoinPool-1-worker-0" daemon prio=10 tid=0x00007f54dcc35000 nid=0x505f waiting on condition [0x00007f54882ce000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f549a393398> (a scala.concurrent.forkjoin.ForkJoinPool$WaitQueueNode)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at scala.concurrent.forkjoin.ForkJoinPool$WaitQueueNode.awaitSyncRelease(ForkJoinPool.java:1340)
at scala.concurrent.forkjoin.ForkJoinPool.sync(ForkJoinPool.java:1417)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.mainLoop(ForkJoinWorkerThread.java:342)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:325)
"Timer-0" daemon prio=10 tid=0x00007f54dcbdc800 nid=0x505e in Object.wait() [0x00007f54883cf000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00007f549a393be0> (a java.util.TaskQueue)
at java.lang.Object.wait(Object.java:485)
at java.util.TimerThread.mainLoop(Timer.java:483)
- locked <0x00007f549a393be0> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:462)
"Low Memory Detector" daemon prio=10 tid=0x00007f54dc08b800 nid=0x4f3c runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"CompilerThread1" daemon prio=10 tid=0x00007f54dc089000 nid=0x4f3b waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"CompilerThread0" daemon prio=10 tid=0x00007f54dc086800 nid=0x4f3a waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Signal Dispatcher" daemon prio=10 tid=0x00007f54dc084000 nid=0x4f39 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" daemon prio=10 tid=0x00007f54dc065000 nid=0x4f38 in Object.wait() [0x00007f54e0280000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00007f5499140240> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
- locked <0x00007f5499140240> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
"Reference Handler" daemon prio=10 tid=0x00007f54dc063000 nid=0x4f37 in Object.wait() [0x00007f54e0381000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00007f5499140320> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:485)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
- locked <0x00007f5499140320> (a java.lang.ref.Reference$Lock)
"main" prio=10 tid=0x00007f54dc007000 nid=0x4f33 waiting on condition [0x00007f54e17f5000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007f54c6327c80> (a org.elasticsearch.common.util.concurrent.AbstractFuture$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:747)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:905)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1217)
at org.elasticsearch.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:249)
at org.elasticsearch.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:78)
at org.elasticsearch.action.support.AdapterActionFuture.actionGet(AdapterActionFuture.java:42)
at com.example.p7.elasticsearch.impl.ElasticSearchClientImpl.index(ElasticSearchClientImpl.scala:131)
at com.example.p7.elasticsearch.impl.IndexImpl.index(IndexImpl.scala:11)
at com.example.p7.elasticsearch.impl.ObjectIndexImpl.indexObject(ObjectIndexImpl.scala:7)
at com.example.p7.cqrs.view.snapshot.ESSnapshotsRepository.storeSnapshot(ESSnapshotsRepository.scala:14)
at com.example.p7.cqrs.repository.RepositoryImpl.com$example$p7$cqrs$repository$RepositoryImpl$$saveAggregate(RepositoryImpl.scala:68)
at com.example.p7.cqrs.repository.RepositoryImpl$$anonfun$save$1.apply(RepositoryImpl.scala:33)
at com.example.p7.cqrs.repository.RepositoryImpl$$anonfun$save$1.apply(RepositoryImpl.scala:33)
at scala.collection.LinearSeqOptimized$class.foreach(LinearSeqOptimized.scala:61)
at scala.collection.immutable.List.foreach(List.scala:45)
at com.example.p7.cqrs.repository.RepositoryImpl.save(RepositoryImpl.scala:33)
at com.example.p7.cqrs.repository.TransactionImpl.commit(TransactionImpl.scala:13)
at com.example.p7.cqrs.bus.LocalInMemoryCommandSender$CommandHandlerActor$$anonfun$processCommand$1.apply(LocalInMemoryCommandSender.scala:96)
at com.example.p7.cqrs.bus.LocalInMemoryCommandSender$CommandHandlerActor$$anonfun$processCommand$1.apply(LocalInMemoryCommandSender.scala:91)
at com.example.p7.cqrs.bus.NoOpTransactionSupport$.transactional(TransactionSupport.scala:12)
at com.example.p7.cqrs.bus.LocalInMemoryCommandSender$CommandHandlerActor.processCommand(LocalInMemoryCommandSender.scala:91)
at com.example.p7.cqrs.bus.LocalInMemoryCommandSender.send(LocalInMemoryCommandSender.scala:50)
at com.example.bss.cpm.pml.executor.impl.PmlCategoryExecutor$class.setProductFamiliesDictionary(PmlCategoryExecutor.scala:97)
at com.example.bss.cpm.pml.executor.PmlExecutor.setProductFamiliesDictionary(PmlExecutor.scala:24)
at com.example.bss.cpm.pml.executor.PmlExecutor.processCommand(PmlExecutor.scala:122)
at com.example.bss.cpm.pml.executor.PmlExecutor.processConstruct$1(PmlExecutor.scala:58)
at com.example.bss.cpm.pml.executor.PmlExecutor$$anonfun$processConstructs$1.apply(PmlExecutor.scala:63)
at com.example.bss.cpm.pml.executor.PmlExecutor$$anonfun$processConstructs$1.apply(PmlExecutor.scala:63)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:206)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:206)
at scala.collection.LinearSeqOptimized$class.foreach(LinearSeqOptimized.scala:61)
at scala.collection.immutable.List.foreach(List.scala:45)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:206)
at scala.collection.immutable.List.map(List.scala:45)
at com.example.bss.cpm.pml.executor.PmlExecutor.processConstructs(PmlExecutor.scala:63)
at com.example.bss.cpm.pml.executor.PmlExecutor.execute(PmlExecutor.scala:46)
at com.example.bss.cpm.tests.pml.PmlSuite.executePmlFile(PmlSuite.scala:46)
at com.example.bss.cpm.tests.pml.InheritedChargesInVariantPmlSuite$$anonfun$3$$anonfun$apply$mcV$sp$3.apply(InheritedChargesInVariantPmlSuite.scala:84)
at com.example.bss.cpm.tests.pml.InheritedChargesInVariantPmlSuite$$anonfun$3$$anonfun$apply$mcV$sp$3.apply(InheritedChargesInVariantPmlSuite.scala:82)
at com.example.bss.cpm.tests.pml.PmlSuite.withConfig(PmlSuite.scala:25)
at com.example.bss.cpm.tests.pml.InheritedChargesInVariantPmlSuite$$anonfun$3.apply$mcV$sp(InheritedChargesInVariantPmlSuite.scala:81)
at com.example.bss.cpm.tests.pml.InheritedChargesInVariantPmlSuite$$anonfun$3.apply(InheritedChargesInVariantPmlSuite.scala:81)
at com.example.bss.cpm.tests.pml.InheritedChargesInVariantPmlSuite$$anonfun$3.apply(InheritedChargesInVariantPmlSuite.scala:81)
at org.scalatest.FunSuite$$anon$2.apply(FunSuite.scala:1146)
at org.scalatest.Suite$class.withFixture(Suite.scala:1477)
at com.example.bss.cpm.tests.pml.PmlSuite.withFixture(PmlSuite.scala:20)
at org.scalatest.FunSuite$class.runTest(FunSuite.scala:1143)
at com.example.bss.cpm.tests.pml.PmlSuite.runTest(PmlSuite.scala:20)
at org.scalatest.FunSuite$$anonfun$runTests$1.apply(FunSuite.scala:1252)
at org.scalatest.FunSuite$$anonfun$runTests$1.apply(FunSuite.scala:1243)
at scala.collection.LinearSeqOptimized$class.foreach(LinearSeqOptimized.scala:61)
at scala.collection.immutable.List.foreach(List.scala:45)
at org.scalatest.FunSuite$class.runTests(FunSuite.scala:1243)
at com.example.bss.cpm.tests.pml.PmlSuite.runTests(PmlSuite.scala:20)
at org.scalatest.Suite$class.run(Suite.scala:1772)
at com.example.bss.cpm.tests.pml.PmlSuite.org$scalatest$FunSuite$$super$run(PmlSuite.scala:20)
at org.scalatest.FunSuite$class.run(FunSuite.scala:1289)
at com.example.bss.cpm.tests.pml.PmlSuite.run(PmlSuite.scala:20)
at org.scalatest.Suite$class.callExecuteOnSuite$1(Suite.scala:1869)
at org.scalatest.Suite$$anonfun$runNestedSuites$1.apply(Suite.scala:1891)
at org.scalatest.Suite$$anonfun$runNestedSuites$1.apply(Suite.scala:1891)
at scala.collection.LinearSeqOptimized$class.foreach(LinearSeqOptimized.scala:61)
at scala.collection.immutable.List.foreach(List.scala:45)
at org.scalatest.Suite$class.runNestedSuites(Suite.scala:1891)
at org.scalatest.tools.DiscoverySuite.runNestedSuites(DiscoverySuite.scala:28)
at org.scalatest.Suite$class.run(Suite.scala:1769)
at org.scalatest.tools.DiscoverySuite.run(DiscoverySuite.scala:28)
at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:59)
at org.scalatest.tools.Runner$$anonfun$doRunRunRunADoRunRun$3.apply(Runner.scala:1515)
at org.scalatest.tools.Runner$$anonfun$doRunRunRunADoRunRun$3.apply(Runner.scala:1512)
at scala.collection.LinearSeqOptimized$class.foreach(LinearSeqOptimized.scala:61)
at scala.collection.immutable.List.foreach(List.scala:45)
at org.scalatest.tools.Runner$.doRunRunRunADoRunRun(Runner.scala:1512)
at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:594)
at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:593)
at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:1556)
at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:592)
at org.scalatest.tools.Runner$.run(Runner.scala:495)
at org.scalatest.tools.Runner.run(Runner.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.scalatest.tools.maven.AbstractScalaTestMojo.runScalaTest(AbstractScalaTestMojo.java:109)
at org.scalatest.tools.maven.TestMojo.execute(TestMojo.java:90)
at org.apache.maven.plugin.DefaultBuildPluginManager.executeMojo(DefaultBuildPluginManager.java:107)
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:209)
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:153)
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:145)
at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:84)
at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:59)
at org.apache.maven.lifecycle.internal.LifecycleStarter.singleThreadedBuild(LifecycleStarter.java:183)
at org.apache.maven.lifecycle.internal.LifecycleStarter.execute(LifecycleStarter.java:161)
at org.apache.maven.DefaultMaven.doExecute(DefaultMaven.java:319)
at org.apache.maven.DefaultMaven.execute(DefaultMaven.java:156)
at org.apache.maven.cli.MavenCli.execute(MavenCli.java:534)
at org.apache.maven.cli.MavenCli.doMain(MavenCli.java:196)
at org.apache.maven.cli.MavenCli.main(MavenCli.java:141)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.codehaus.plexus.classworlds.launcher.Launcher.launchEnhanced(Launcher.java:290)
at org.codehaus.plexus.classworlds.launcher.Launcher.launch(Launcher.java:230)
at org.codehaus.plexus.classworlds.launcher.Launcher.mainWithExitCode(Launcher.java:409)
at org.codehaus.plexus.classworlds.launcher.Launcher.main(Launcher.java:352)
"VM Thread" prio=10 tid=0x00007f54dc05e800 nid=0x4f36 runnable
"GC task thread#0 (ParallelGC)" prio=10 tid=0x00007f54dc01a000 nid=0x4f34 runnable
"GC task thread#1 (ParallelGC)" prio=10 tid=0x00007f54dc01b800 nid=0x4f35 runnable
"VM Periodic Task Thread" prio=10 tid=0x00007f54dc08e000 nid=0x4f3d waiting on condition
JNI global references: 865
Found one Java-level deadlock:
=============================
"elasticsearch[index]-pool-180-thread-1":
waiting to lock monitor 0x00007f54840147c0 (object 0x00007f54c5133670, a java.lang.Object),
which is held by "elasticsearch[Rhiannon]clusterService#updateTask-pool-189-thread-1"
"elasticsearch[Rhiannon]clusterService#updateTask-pool-189-thread-1":
waiting to lock monitor 0x00007f5484014868 (object 0x00007f54c620bf28, a java.lang.Object),
which is held by "elasticsearch[index]-pool-180-thread-1"
Java stack information for the threads listed above:
===================================================
"elasticsearch[index]-pool-180-thread-1":
at org.elasticsearch.index.mapper.MapperService$InternalFieldMapperListener.fieldMapper(MapperService.java:710)
- waiting to lock <0x00007f54c5133670> (a java.lang.Object)
at org.elasticsearch.index.mapper.DocumentMapper.addFieldMapper(DocumentMapper.java:634)
- locked <0x00007f54c620bf28> (a java.lang.Object)
at org.elasticsearch.index.mapper.object.ObjectMapper$3.fieldMapper(ObjectMapper.java:695)
at org.elasticsearch.index.mapper.core.AbstractFieldMapper.traverse(AbstractFieldMapper.java:323)
at org.elasticsearch.index.mapper.object.ObjectMapper.serializeValue(ObjectMapper.java:693)
- locked <0x00007f54c620b9c0> (a java.lang.Object)
at org.elasticsearch.index.mapper.object.ObjectMapper.parse(ObjectMapper.java:440)
at org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:566)
at org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:490)
at org.elasticsearch.index.shard.service.InternalIndexShard.prepareIndex(InternalIndexShard.java:289)
at org.elasticsearch.action.index.TransportIndexAction.shardOperationOnPrimary(TransportIndexAction.java:185)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:428)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$1.run(TransportShardReplicationOperationAction.java:341)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
"elasticsearch[Rhiannon]clusterService#updateTask-pool-189-thread-1":
at org.elasticsearch.index.mapper.DocumentMapper.addObjectMapperListener(DocumentMapper.java:670)
- waiting to lock <0x00007f54c620bf28> (a java.lang.Object)
at org.elasticsearch.index.mapper.MapperService.add(MapperService.java:187)
- locked <0x00007f54c5133670> (a java.lang.Object)
at org.elasticsearch.index.mapper.MapperService.add(MapperService.java:166)
at org.elasticsearch.indices.cluster.IndicesClusterStateService.processMapping(IndicesClusterStateService.java:382)
at org.elasticsearch.indices.cluster.IndicesClusterStateService.applyMappings(IndicesClusterStateService.java:349)
at org.elasticsearch.indices.cluster.IndicesClusterStateService.clusterChanged(IndicesClusterStateService.java:176)
- locked <0x00007f54c4a1f168> (a java.lang.Object)
at org.elasticsearch.cluster.service.InternalClusterService$2.run(InternalClusterService.java:254)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
Found 1 deadlock.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment