Created
September 12, 2011 18:59
-
-
Save jeffstyr/1212071 to your computer and use it in GitHub Desktop.
QueueCollection + PersistentQueue deadlock
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Found one Java-level deadlock: | |
| ============================= | |
| "New I/O server worker #1-48": | |
| waiting to lock monitor 0x0000000059f639b0 (object 0x000000072bba4d90, a net.lag.kestrel.QueueCollection), | |
| which is held by "Hashed wheel timer #1" | |
| "Hashed wheel timer #1": | |
| waiting to lock monitor 0x000000005a9c4378 (object 0x000000072bb8c7a0, a net.lag.kestrel.PersistentQueue), | |
| which is held by "New I/O server worker #1-41" | |
| "New I/O server worker #1-41": | |
| waiting to lock monitor 0x0000000059f639b0 (object 0x000000072bba4d90, a net.lag.kestrel.QueueCollection), | |
| which is held by "Hashed wheel timer #1" | |
| Java stack information for the threads listed above: | |
| =================================================== | |
| "New I/O server worker #1-48": | |
| at net.lag.kestrel.QueueCollection.queue(QueueCollection.scala:89) | |
| - waiting to lock <0x000000072bba4d90> (a net.lag.kestrel.QueueCollection) | |
| at net.lag.kestrel.QueueCollection.add(QueueCollection.scala:123) | |
| at net.lag.kestrel.KestrelHandler.setItem(KestrelHandler.scala:190) | |
| at net.lag.kestrel.MemcacheHandler.handle(MemcacheHandler.scala:62) | |
| at net.lag.kestrel.MemcacheHandler.handle(MemcacheHandler.scala:33) | |
| at net.lag.kestrel.NettyHandler.handleUpstream(NettyHandler.scala:53) | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:545) | |
| at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:754) | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:302) | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:317) | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:299) | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:216) | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:80) | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:545) | |
| at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:754) | |
| at org.jboss.netty.handler.timeout.IdleStateHandler.messageReceived(IdleStateHandler.java:276) | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:80) | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:545) | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:540) | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:274) | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:261) | |
| at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:350) | |
| at org.jboss.netty.channel.socket.nio.NioWorker.processSelectedKeys(NioWorker.java:281) | |
| at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:201) | |
| at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.jboss.netty.util.internal.IoWorkerRunnable.run(IoWorkerRunnable.java:46) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) | |
| at java.lang.Thread.run(Thread.java:662) | |
| "Hashed wheel timer #1": | |
| at net.lag.kestrel.PersistentQueue.discardExpired(PersistentQueue.scala:445) | |
| - waiting to lock <0x000000072bb8c7a0> (a net.lag.kestrel.PersistentQueue) | |
| at net.lag.kestrel.QueueCollection$$anonfun$flushExpired$2.apply(QueueCollection.scala:191) | |
| at net.lag.kestrel.QueueCollection$$anonfun$flushExpired$2.apply(QueueCollection.scala:191) | |
| at scala.Option.map(Option.scala:129) | |
| at net.lag.kestrel.QueueCollection.flushExpired(QueueCollection.scala:191) | |
| - locked <0x000000072bba4d90> (a net.lag.kestrel.QueueCollection) | |
| at net.lag.kestrel.QueueCollection$$anonfun$flushAllExpired$1.apply(QueueCollection.scala:196) | |
| at net.lag.kestrel.QueueCollection$$anonfun$flushAllExpired$1.apply(QueueCollection.scala:196) | |
| at scala.collection.LinearSeqOptimized$class.foldLeft(LinearSeqOptimized.scala:123) | |
| at scala.collection.immutable.List.foldLeft(List.scala:45) | |
| at net.lag.kestrel.QueueCollection.flushAllExpired(QueueCollection.scala:196) | |
| - locked <0x000000072bba4d90> (a net.lag.kestrel.QueueCollection) | |
| at net.lag.kestrel.Kestrel$$anon$5.run(Kestrel.scala:136) | |
| at org.jboss.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:504) | |
| at org.jboss.netty.util.HashedWheelTimer$Worker.notifyExpiredTimeouts(HashedWheelTimer.java:419) | |
| at org.jboss.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:369) | |
| at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at java.lang.Thread.run(Thread.java:662) | |
| "New I/O server worker #1-41": | |
| at net.lag.kestrel.QueueCollection.queue(QueueCollection.scala:89) | |
| - waiting to lock <0x000000072bba4d90> (a net.lag.kestrel.QueueCollection) | |
| at net.lag.kestrel.QueueCollection.unremove(QueueCollection.scala:161) | |
| at net.lag.kestrel.KestrelHandler$pendingTransactions$$anonfun$cancelAll$1$$anonfun$apply$1.apply$mcVI$sp(KestrelHandler.scala:72) | |
| at net.lag.kestrel.KestrelHandler$pendingTransactions$$anonfun$cancelAll$1$$anonfun$apply$1.apply(KestrelHandler.scala:72) | |
| at net.lag.kestrel.KestrelHandler$pendingTransactions$$anonfun$cancelAll$1$$anonfun$apply$1.apply(KestrelHandler.scala:72) | |
| at scala.collection.LinearSeqOptimized$class.foreach(LinearSeqOptimized.scala:61) | |
| at scala.collection.immutable.List.foreach(List.scala:45) | |
| at scala.collection.generic.TraversableForwarder$class.foreach(TraversableForwarder.scala:44) | |
| at scala.collection.mutable.ListBuffer.foreach(ListBuffer.scala:42) | |
| at net.lag.kestrel.KestrelHandler$pendingTransactions$$anonfun$cancelAll$1.apply(KestrelHandler.scala:72) | |
| at net.lag.kestrel.KestrelHandler$pendingTransactions$$anonfun$cancelAll$1.apply(KestrelHandler.scala:71) | |
| at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:80) | |
| at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:80) | |
| at scala.collection.Iterator$class.foreach(Iterator.scala:631) | |
| at scala.collection.mutable.HashTable$$anon$1.foreach(HashTable.scala:161) | |
| at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:194) | |
| at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:39) | |
| at scala.collection.mutable.HashMap.foreach(HashMap.scala:80) | |
| at net.lag.kestrel.KestrelHandler$pendingTransactions$.cancelAll(KestrelHandler.scala:71) | |
| - locked <0x000000072c5a1468> (a net.lag.kestrel.KestrelHandler$pendingTransactions$) | |
| at net.lag.kestrel.KestrelHandler.abortAnyTransaction(KestrelHandler.scala:184) | |
| at net.lag.kestrel.KestrelHandler.finish(KestrelHandler.scala:95) | |
| at net.lag.kestrel.NettyHandler.handleUpstream(NettyHandler.scala:59) | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:545) | |
| at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:754) | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.exceptionCaught(FrameDecoder.java:238) | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:122) | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:545) | |
| at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:754) | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.exceptionCaught(SimpleChannelUpstreamHandler.java:148) | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:122) | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:545) | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:540) | |
| at org.jboss.netty.channel.Channels.fireExceptionCaught(Channels.java:432) | |
| at org.jboss.netty.channel.socket.nio.NioWorker.cleanUpWriteBuffer(NioWorker.java:653) | |
| at org.jboss.netty.channel.socket.nio.NioWorker.writeFromUserCode(NioWorker.java:370) | |
| at org.jboss.netty.channel.socket.nio.NioServerSocketPipelineSink.handleAcceptedSocket(NioServerSocketPipelineSink.java:137) | |
| at org.jboss.netty.channel.socket.nio.NioServerSocketPipelineSink.eventSunk(NioServerSocketPipelineSink.java:76) | |
| at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendDownstream(DefaultChannelPipeline.java:742) | |
| at org.jboss.netty.channel.Channels.write(Channels.java:632) | |
| at com.twitter.naggati.Codec.handleDownstream(Codec.scala:65) | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:568) | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:563) | |
| at org.jboss.netty.channel.Channels.write(Channels.java:611) | |
| at org.jboss.netty.channel.Channels.write(Channels.java:578) | |
| at org.jboss.netty.channel.AbstractChannel.write(AbstractChannel.java:259) | |
| at net.lag.kestrel.MemcacheHandler$$anonfun$get$2.apply(MemcacheHandler.scala:162) | |
| at net.lag.kestrel.MemcacheHandler$$anonfun$get$2.apply(MemcacheHandler.scala:157) | |
| at com.twitter.util.Return$$anonfun$map$1.apply(Try.scala:163) | |
| at com.twitter.util.Try$.apply(Try.scala:13) | |
| at com.twitter.util.Return.map(Try.scala:163) | |
| at com.twitter.util.Return.map(Try.scala:157) | |
| at com.twitter.util.Promise$$anon$1$$anonfun$7.apply(Future.scala:445) | |
| at com.twitter.util.Promise$$anon$1$$anonfun$7.apply(Future.scala:444) | |
| at com.twitter.util.Promise$$anonfun$respond$1.apply(Future.scala:425) | |
| at com.twitter.util.Promise$$anonfun$respond$1.apply(Future.scala:421) | |
| at com.twitter.concurrent.IVar.set(IVar.scala:50) | |
| at com.twitter.util.Promise.updateIfEmpty(Future.scala:416) | |
| at com.twitter.util.Promise.update(Future.scala:404) | |
| at com.twitter.util.Promise$$anon$1$$anonfun$7.apply(Future.scala:445) | |
| at com.twitter.util.Promise$$anon$1$$anonfun$7.apply(Future.scala:444) | |
| at com.twitter.util.Promise$$anonfun$respond$1.apply(Future.scala:425) | |
| at com.twitter.util.Promise$$anonfun$respond$1.apply(Future.scala:421) | |
| at com.twitter.concurrent.IVar.set(IVar.scala:50) | |
| at com.twitter.util.Promise.updateIfEmpty(Future.scala:416) | |
| at com.twitter.util.Promise.update(Future.scala:404) | |
| at com.twitter.util.Promise$$anon$1$$anonfun$7.apply(Future.scala:445) | |
| at com.twitter.util.Promise$$anon$1$$anonfun$7.apply(Future.scala:444) | |
| at com.twitter.util.Promise$$anonfun$respond$1.apply(Future.scala:425) | |
| at com.twitter.util.Promise$$anonfun$respond$1.apply(Future.scala:421) | |
| at com.twitter.concurrent.IVar.set(IVar.scala:50) | |
| at com.twitter.util.Promise.updateIfEmpty(Future.scala:416) | |
| at com.twitter.util.Promise.update(Future.scala:404) | |
| at com.twitter.util.Promise.setValue(Future.scala:387) | |
| at net.lag.kestrel.PersistentQueue.net$lag$kestrel$PersistentQueue$$waitOperation(PersistentQueue.scala:266) | |
| at net.lag.kestrel.PersistentQueue$$anonfun$net$lag$kestrel$PersistentQueue$$waitOperation$1.apply$mcV$sp(PersistentQueue.scala:261) | |
| at net.lag.kestrel.DeadlineWaitQueue$$anonfun$trigger$2.apply(DeadlineWaitQueue.scala:49) | |
| at net.lag.kestrel.DeadlineWaitQueue$$anonfun$trigger$2.apply(DeadlineWaitQueue.scala:49) | |
| at scala.Option.foreach(Option.scala:185) | |
| at net.lag.kestrel.DeadlineWaitQueue.trigger(DeadlineWaitQueue.scala:49) | |
| at net.lag.kestrel.PersistentQueue.itemWasAdded(PersistentQueue.scala:207) | |
| at net.lag.kestrel.PersistentQueue.add(PersistentQueue.scala:200) | |
| at net.lag.kestrel.PersistentQueue.add(PersistentQueue.scala:211) | |
| at net.lag.kestrel.PersistentQueue$$anonfun$discardExpired$1$$anonfun$apply$2.apply(PersistentQueue.scala:472) | |
| at net.lag.kestrel.PersistentQueue$$anonfun$discardExpired$1$$anonfun$apply$2.apply(PersistentQueue.scala:472) | |
| at scala.collection.LinearSeqOptimized$class.foreach(LinearSeqOptimized.scala:61) | |
| at scala.collection.immutable.List.foreach(List.scala:45) | |
| at scala.collection.generic.TraversableForwarder$class.foreach(TraversableForwarder.scala:44) | |
| at scala.collection.mutable.ListBuffer.foreach(ListBuffer.scala:42) | |
| at net.lag.kestrel.PersistentQueue$$anonfun$discardExpired$1.apply(PersistentQueue.scala:472) | |
| at net.lag.kestrel.PersistentQueue$$anonfun$discardExpired$1.apply(PersistentQueue.scala:471) | |
| at scala.Option.foreach(Option.scala:185) | |
| at net.lag.kestrel.PersistentQueue.discardExpired(PersistentQueue.scala:471) | |
| at net.lag.kestrel.PersistentQueue.net$lag$kestrel$PersistentQueue$$_add(PersistentQueue.scala:410) | |
| at net.lag.kestrel.PersistentQueue.add(PersistentQueue.scala:190) | |
| - locked <0x000000072bb8c7a0> (a net.lag.kestrel.PersistentQueue) | |
| at net.lag.kestrel.PersistentQueue.add(PersistentQueue.scala:211) | |
| at net.lag.kestrel.QueueCollection.add(QueueCollection.scala:126) | |
| at net.lag.kestrel.KestrelHandler.setItem(KestrelHandler.scala:190) | |
| at net.lag.kestrel.MemcacheHandler.handle(MemcacheHandler.scala:62) | |
| at net.lag.kestrel.MemcacheHandler.handle(MemcacheHandler.scala:33) | |
| at net.lag.kestrel.NettyHandler.handleUpstream(NettyHandler.scala:53) | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:545) | |
| at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:754) | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:302) | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:317) | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:299) | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:216) | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:80) | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:545) | |
| at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:754) | |
| at org.jboss.netty.handler.timeout.IdleStateHandler.messageReceived(IdleStateHandler.java:276) | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:80) | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:545) | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:540) | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:274) | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:261) | |
| at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:350) | |
| at org.jboss.netty.channel.socket.nio.NioWorker.processSelectedKeys(NioWorker.java:281) | |
| at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:201) | |
| at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.jboss.netty.util.internal.IoWorkerRunnable.run(IoWorkerRunnable.java:46) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) | |
| at java.lang.Thread.run(Thread.java:662) | |
| Found 1 deadlock. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment