Skip to content

Instantly share code, notes, and snippets.

@hrsakai
Created August 3, 2018 01:10
Show Gist options
  • Save hrsakai/d65e8e2cd511173232b1010a9bbdf126 to your computer and use it in GitHub Desktop.
Save hrsakai/d65e8e2cd511173232b1010a9bbdf126 to your computer and use it in GitHub Desktop.
Full thread dump OpenJDK 64-Bit Server VM (25.171-b10 mixed mode):
"Attach Listener" #81 daemon prio=9 os_prio=0 tid=0x00007f04f8000b70 nid=0x3dde waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"bookkeeper-io-3-16" #80 prio=5 os_prio=0 tid=0x00007f04b4010bf0 nid=0x3fde runnable [0x00007f04750cd000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-3-15" #79 prio=5 os_prio=0 tid=0x00007f04b400ee10 nid=0x3fdd runnable [0x00007f04753ce000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-3-14" #78 prio=5 os_prio=0 tid=0x00007f04b400d8a0 nid=0x3fdc runnable [0x00007f04754cf000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-3-13" #77 prio=5 os_prio=0 tid=0x00007f04b400c390 nid=0x3fdb runnable [0x00007f04755d0000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-3-12" #76 prio=5 os_prio=0 tid=0x00007f04b400b040 nid=0x3fda runnable [0x00007f04756d1000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-3-11" #75 prio=5 os_prio=0 tid=0x00007f04b4021f80 nid=0x3fd9 runnable [0x00007f04757d2000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-3-10" #74 prio=5 os_prio=0 tid=0x00007f04b4020a90 nid=0x3fd8 runnable [0x00007f04758d3000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-3-9" #73 prio=5 os_prio=0 tid=0x00007f04b401f5a0 nid=0x3fd7 runnable [0x00007f04759d4000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-3-8" #72 prio=5 os_prio=0 tid=0x00007f04b401e0b0 nid=0x3fd6 runnable [0x00007f0475ad5000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-3-7" #71 prio=5 os_prio=0 tid=0x00007f04b401b970 nid=0x3fd5 runnable [0x00007f0475bd6000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-3-6" #70 prio=5 os_prio=0 tid=0x00007f04b404d320 nid=0x3fd4 runnable [0x00007f0475cd7000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-3-5" #68 prio=5 os_prio=0 tid=0x00007f04a8000cb0 nid=0x3d4f runnable [0x00007f0475dd8000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-3-4" #67 prio=5 os_prio=0 tid=0x00007f04900161d0 nid=0x3cee runnable [0x00007f0475ed9000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-3-3" #66 prio=5 os_prio=0 tid=0x00007f04b0016cf0 nid=0x3cbf runnable [0x00007f0475fda000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-3-2" #65 prio=5 os_prio=0 tid=0x00007f049001af50 nid=0x3cbe runnable [0x00007f04760db000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-3-1" #64 prio=5 os_prio=0 tid=0x00007f0490019920 nid=0x3cbd runnable [0x00007f04761dc000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-19-5" #63 prio=5 os_prio=0 tid=0x00007f04c8070360 nid=0x3cbc runnable [0x00007f04764dd000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-19-4" #62 prio=5 os_prio=0 tid=0x00007f04c806eed0 nid=0x3cbb runnable [0x00007f04765de000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-19-3" #61 prio=5 os_prio=0 tid=0x00007f04c806da60 nid=0x3cba runnable [0x00007f04766df000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-19-2" #60 prio=5 os_prio=0 tid=0x00007f04c806c850 nid=0x3cb9 runnable [0x00007f04767e0000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"bookkeeper-io-19-1" #59 prio=5 os_prio=0 tid=0x00007f04c8056980 nid=0x3cb8 runnable [0x00007f04768e1000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:114)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:235)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:252)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"ObjectCleanerThread" #58 daemon prio=1 os_prio=0 tid=0x00007f04c80ef210 nid=0x3cb7 in Object.wait() [0x00007f0476dec000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
- locked <0x00000000e05710e0> (a java.lang.ref.ReferenceQueue$Lock)
at io.netty.util.internal.ObjectCleaner$1.run(ObjectCleaner.java:54)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"ZkLedgerManagerScheduler-20-1" #57 prio=5 os_prio=0 tid=0x00007f044c000bb0 nid=0x3cb6 waiting on condition [0x00007f0476eed000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e04e6e60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-7-0" #55 prio=5 os_prio=0 tid=0x00007f0468005980 nid=0x3cb3 waiting on condition [0x00007f0476fee000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e137bd28> (a java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
at org.apache.bookkeeper.util.ZkUtils.createFullPathOptimistic(ZkUtils.java:184)
at org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager.markLedgerUnderreplicated(ZkLedgerUnderreplicationManager.java:269)
at org.apache.bookkeeper.replication.Auditor.publishSuspectedLedgers(Auditor.java:550)
at org.apache.bookkeeper.replication.Auditor.access$1400(Auditor.java:79)
at org.apache.bookkeeper.replication.Auditor$ProcessLostFragmentsCb.operationComplete(Auditor.java:580)
at org.apache.bookkeeper.replication.Auditor$ProcessLostFragmentsCb.operationComplete(Auditor.java:563)
at org.apache.bookkeeper.client.LedgerChecker$FullLedgerCallback.operationComplete(LedgerChecker.java:303)
at org.apache.bookkeeper.client.LedgerChecker$FullLedgerCallback.operationComplete(LedgerChecker.java:282)
at org.apache.bookkeeper.client.LedgerChecker$LedgerFragmentCallback.operationComplete(LedgerChecker.java:130)
at org.apache.bookkeeper.client.LedgerChecker$LedgerFragmentCallback.operationComplete(LedgerChecker.java:91)
at org.apache.bookkeeper.client.LedgerChecker$ReadManyEntriesCallback.readEntryComplete(LedgerChecker.java:83)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion$1.readEntryComplete(PerChannelBookieClient.java:1559)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion.lambda$errorOut$0(PerChannelBookieClient.java:1575)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion$$Lambda$28/167038546.run(Unknown Source)
at org.apache.bookkeeper.proto.PerChannelBookieClient$CompletionValue$1.safeRun(PerChannelBookieClient.java:1417)
at org.apache.bookkeeper.common.util.SafeRunnable.run(SafeRunnable.java:36)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-6-0" #54 prio=5 os_prio=0 tid=0x00007f0468004080 nid=0x3cb2 waiting on condition [0x00007f04770ef000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e1379238> (a java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
at org.apache.bookkeeper.util.ZkUtils.createFullPathOptimistic(ZkUtils.java:184)
at org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager.markLedgerUnderreplicated(ZkLedgerUnderreplicationManager.java:269)
at org.apache.bookkeeper.replication.Auditor.publishSuspectedLedgers(Auditor.java:550)
at org.apache.bookkeeper.replication.Auditor.access$1400(Auditor.java:79)
at org.apache.bookkeeper.replication.Auditor$ProcessLostFragmentsCb.operationComplete(Auditor.java:580)
at org.apache.bookkeeper.replication.Auditor$ProcessLostFragmentsCb.operationComplete(Auditor.java:563)
at org.apache.bookkeeper.client.LedgerChecker$FullLedgerCallback.operationComplete(LedgerChecker.java:303)
at org.apache.bookkeeper.client.LedgerChecker$FullLedgerCallback.operationComplete(LedgerChecker.java:282)
at org.apache.bookkeeper.client.LedgerChecker$LedgerFragmentCallback.operationComplete(LedgerChecker.java:130)
at org.apache.bookkeeper.client.LedgerChecker$LedgerFragmentCallback.operationComplete(LedgerChecker.java:91)
at org.apache.bookkeeper.client.LedgerChecker$ReadManyEntriesCallback.readEntryComplete(LedgerChecker.java:83)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion$1.readEntryComplete(PerChannelBookieClient.java:1559)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion.lambda$errorOut$0(PerChannelBookieClient.java:1575)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion$$Lambda$28/167038546.run(Unknown Source)
at org.apache.bookkeeper.proto.PerChannelBookieClient$CompletionValue$1.safeRun(PerChannelBookieClient.java:1417)
at org.apache.bookkeeper.common.util.SafeRunnable.run(SafeRunnable.java:36)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-5-0" #53 prio=5 os_prio=0 tid=0x00007f0468002bd0 nid=0x3cb1 waiting on condition [0x00007f04771f0000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e137b988> (a java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
at org.apache.bookkeeper.util.ZkUtils.createFullPathOptimistic(ZkUtils.java:184)
at org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager.markLedgerUnderreplicated(ZkLedgerUnderreplicationManager.java:269)
at org.apache.bookkeeper.replication.Auditor.publishSuspectedLedgers(Auditor.java:550)
at org.apache.bookkeeper.replication.Auditor.access$1400(Auditor.java:79)
at org.apache.bookkeeper.replication.Auditor$ProcessLostFragmentsCb.operationComplete(Auditor.java:580)
at org.apache.bookkeeper.replication.Auditor$ProcessLostFragmentsCb.operationComplete(Auditor.java:563)
at org.apache.bookkeeper.client.LedgerChecker$FullLedgerCallback.operationComplete(LedgerChecker.java:303)
at org.apache.bookkeeper.client.LedgerChecker$FullLedgerCallback.operationComplete(LedgerChecker.java:282)
at org.apache.bookkeeper.client.LedgerChecker$LedgerFragmentCallback.operationComplete(LedgerChecker.java:130)
at org.apache.bookkeeper.client.LedgerChecker$LedgerFragmentCallback.operationComplete(LedgerChecker.java:91)
at org.apache.bookkeeper.client.LedgerChecker$ReadManyEntriesCallback.readEntryComplete(LedgerChecker.java:83)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion$1.readEntryComplete(PerChannelBookieClient.java:1559)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion.lambda$errorOut$0(PerChannelBookieClient.java:1575)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion$$Lambda$28/167038546.run(Unknown Source)
at org.apache.bookkeeper.proto.PerChannelBookieClient$CompletionValue$1.safeRun(PerChannelBookieClient.java:1417)
at org.apache.bookkeeper.common.util.SafeRunnable.run(SafeRunnable.java:36)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-4-0" #52 prio=5 os_prio=0 tid=0x00007f04680250c0 nid=0x3cb0 waiting on condition [0x00007f04772f1000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e1378758> (a java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
at org.apache.bookkeeper.util.ZkUtils.createFullPathOptimistic(ZkUtils.java:184)
at org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager.markLedgerUnderreplicated(ZkLedgerUnderreplicationManager.java:269)
at org.apache.bookkeeper.replication.Auditor.publishSuspectedLedgers(Auditor.java:550)
at org.apache.bookkeeper.replication.Auditor.access$1400(Auditor.java:79)
at org.apache.bookkeeper.replication.Auditor$ProcessLostFragmentsCb.operationComplete(Auditor.java:580)
at org.apache.bookkeeper.replication.Auditor$ProcessLostFragmentsCb.operationComplete(Auditor.java:563)
at org.apache.bookkeeper.client.LedgerChecker$FullLedgerCallback.operationComplete(LedgerChecker.java:303)
at org.apache.bookkeeper.client.LedgerChecker$FullLedgerCallback.operationComplete(LedgerChecker.java:282)
at org.apache.bookkeeper.client.LedgerChecker$LedgerFragmentCallback.operationComplete(LedgerChecker.java:130)
at org.apache.bookkeeper.client.LedgerChecker$LedgerFragmentCallback.operationComplete(LedgerChecker.java:91)
at org.apache.bookkeeper.client.LedgerChecker$ReadManyEntriesCallback.readEntryComplete(LedgerChecker.java:83)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion$1.readEntryComplete(PerChannelBookieClient.java:1559)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion.lambda$errorOut$0(PerChannelBookieClient.java:1575)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion$$Lambda$28/167038546.run(Unknown Source)
at org.apache.bookkeeper.proto.PerChannelBookieClient$CompletionValue$1.safeRun(PerChannelBookieClient.java:1417)
at org.apache.bookkeeper.common.util.SafeRunnable.run(SafeRunnable.java:36)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-3-0" #51 prio=5 os_prio=0 tid=0x00007f04680237c0 nid=0x3caf waiting on condition [0x00007f04773f2000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e1378e98> (a java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
at org.apache.bookkeeper.util.ZkUtils.createFullPathOptimistic(ZkUtils.java:184)
at org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager.markLedgerUnderreplicated(ZkLedgerUnderreplicationManager.java:269)
at org.apache.bookkeeper.replication.Auditor.publishSuspectedLedgers(Auditor.java:550)
at org.apache.bookkeeper.replication.Auditor.access$1400(Auditor.java:79)
at org.apache.bookkeeper.replication.Auditor$ProcessLostFragmentsCb.operationComplete(Auditor.java:580)
at org.apache.bookkeeper.replication.Auditor$ProcessLostFragmentsCb.operationComplete(Auditor.java:563)
at org.apache.bookkeeper.client.LedgerChecker$FullLedgerCallback.operationComplete(LedgerChecker.java:303)
at org.apache.bookkeeper.client.LedgerChecker$FullLedgerCallback.operationComplete(LedgerChecker.java:282)
at org.apache.bookkeeper.client.LedgerChecker$LedgerFragmentCallback.operationComplete(LedgerChecker.java:130)
at org.apache.bookkeeper.client.LedgerChecker$LedgerFragmentCallback.operationComplete(LedgerChecker.java:91)
at org.apache.bookkeeper.client.LedgerChecker$ReadManyEntriesCallback.readEntryComplete(LedgerChecker.java:83)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion$1.readEntryComplete(PerChannelBookieClient.java:1559)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion.lambda$errorOut$0(PerChannelBookieClient.java:1575)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion$$Lambda$28/167038546.run(Unknown Source)
at org.apache.bookkeeper.proto.PerChannelBookieClient$CompletionValue$1.safeRun(PerChannelBookieClient.java:1417)
at org.apache.bookkeeper.common.util.SafeRunnable.run(SafeRunnable.java:36)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-2-0" #50 prio=5 os_prio=0 tid=0x00007f04680222e0 nid=0x3cae waiting on condition [0x00007f04774f3000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e137af78> (a java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
at org.apache.bookkeeper.util.ZkUtils.createFullPathOptimistic(ZkUtils.java:184)
at org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager.markLedgerUnderreplicated(ZkLedgerUnderreplicationManager.java:269)
at org.apache.bookkeeper.replication.Auditor.publishSuspectedLedgers(Auditor.java:550)
at org.apache.bookkeeper.replication.Auditor.access$1400(Auditor.java:79)
at org.apache.bookkeeper.replication.Auditor$ProcessLostFragmentsCb.operationComplete(Auditor.java:580)
at org.apache.bookkeeper.replication.Auditor$ProcessLostFragmentsCb.operationComplete(Auditor.java:563)
at org.apache.bookkeeper.client.LedgerChecker$FullLedgerCallback.operationComplete(LedgerChecker.java:303)
at org.apache.bookkeeper.client.LedgerChecker$FullLedgerCallback.operationComplete(LedgerChecker.java:282)
at org.apache.bookkeeper.client.LedgerChecker$LedgerFragmentCallback.operationComplete(LedgerChecker.java:130)
at org.apache.bookkeeper.client.LedgerChecker$LedgerFragmentCallback.operationComplete(LedgerChecker.java:91)
at org.apache.bookkeeper.client.LedgerChecker$ReadManyEntriesCallback.readEntryComplete(LedgerChecker.java:83)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion$1.readEntryComplete(PerChannelBookieClient.java:1559)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion.lambda$errorOut$0(PerChannelBookieClient.java:1575)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion$$Lambda$28/167038546.run(Unknown Source)
at org.apache.bookkeeper.proto.PerChannelBookieClient$CompletionValue$1.safeRun(PerChannelBookieClient.java:1417)
at org.apache.bookkeeper.common.util.SafeRunnable.run(SafeRunnable.java:36)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-1-0" #49 prio=5 os_prio=0 tid=0x00007f0468020a70 nid=0x3cad waiting on condition [0x00007f04775f4000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e1378af8> (a java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
at org.apache.bookkeeper.util.ZkUtils.createFullPathOptimistic(ZkUtils.java:184)
at org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager.markLedgerUnderreplicated(ZkLedgerUnderreplicationManager.java:269)
at org.apache.bookkeeper.replication.Auditor.publishSuspectedLedgers(Auditor.java:550)
at org.apache.bookkeeper.replication.Auditor.access$1400(Auditor.java:79)
at org.apache.bookkeeper.replication.Auditor$ProcessLostFragmentsCb.operationComplete(Auditor.java:580)
at org.apache.bookkeeper.replication.Auditor$ProcessLostFragmentsCb.operationComplete(Auditor.java:563)
at org.apache.bookkeeper.client.LedgerChecker$FullLedgerCallback.operationComplete(LedgerChecker.java:303)
at org.apache.bookkeeper.client.LedgerChecker$FullLedgerCallback.operationComplete(LedgerChecker.java:282)
at org.apache.bookkeeper.client.LedgerChecker$LedgerFragmentCallback.operationComplete(LedgerChecker.java:130)
at org.apache.bookkeeper.client.LedgerChecker$LedgerFragmentCallback.operationComplete(LedgerChecker.java:91)
at org.apache.bookkeeper.client.LedgerChecker$ReadManyEntriesCallback.readEntryComplete(LedgerChecker.java:83)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion$1.readEntryComplete(PerChannelBookieClient.java:1559)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion.lambda$errorOut$0(PerChannelBookieClient.java:1575)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion$$Lambda$28/167038546.run(Unknown Source)
at org.apache.bookkeeper.proto.PerChannelBookieClient$CompletionValue$1.safeRun(PerChannelBookieClient.java:1417)
at org.apache.bookkeeper.common.util.SafeRunnable.run(SafeRunnable.java:36)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-0-0" #48 prio=5 os_prio=0 tid=0x00007f046800d730 nid=0x3cac waiting on condition [0x00007f04776f5000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e13795d8> (a java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
at org.apache.bookkeeper.util.ZkUtils.createFullPathOptimistic(ZkUtils.java:184)
at org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager.markLedgerUnderreplicated(ZkLedgerUnderreplicationManager.java:269)
at org.apache.bookkeeper.replication.Auditor.publishSuspectedLedgers(Auditor.java:550)
at org.apache.bookkeeper.replication.Auditor.access$1400(Auditor.java:79)
at org.apache.bookkeeper.replication.Auditor$ProcessLostFragmentsCb.operationComplete(Auditor.java:580)
at org.apache.bookkeeper.replication.Auditor$ProcessLostFragmentsCb.operationComplete(Auditor.java:563)
at org.apache.bookkeeper.client.LedgerChecker$FullLedgerCallback.operationComplete(LedgerChecker.java:303)
at org.apache.bookkeeper.client.LedgerChecker$FullLedgerCallback.operationComplete(LedgerChecker.java:282)
at org.apache.bookkeeper.client.LedgerChecker$LedgerFragmentCallback.operationComplete(LedgerChecker.java:130)
at org.apache.bookkeeper.client.LedgerChecker$LedgerFragmentCallback.operationComplete(LedgerChecker.java:91)
at org.apache.bookkeeper.client.LedgerChecker$ReadManyEntriesCallback.readEntryComplete(LedgerChecker.java:83)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion$1.readEntryComplete(PerChannelBookieClient.java:1559)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion.lambda$errorOut$0(PerChannelBookieClient.java:1575)
at org.apache.bookkeeper.proto.PerChannelBookieClient$ReadCompletion$$Lambda$28/167038546.run(Unknown Source)
at org.apache.bookkeeper.proto.PerChannelBookieClient$CompletionValue$1.safeRun(PerChannelBookieClient.java:1417)
at org.apache.bookkeeper.common.util.SafeRunnable.run(SafeRunnable.java:36)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientScheduler-OrderedScheduler-0-0" #47 prio=5 os_prio=0 tid=0x00007f046801deb0 nid=0x3cab waiting on condition [0x00007f04777f6000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e04e4050> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"AuditorBookie-172.17.206.8:3181-EventThread" #46 daemon prio=5 os_prio=0 tid=0x00007f0468013840 nid=0x3caa waiting on condition [0x00007f04778f7000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e04fa9d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
"AuditorBookie-172.17.206.8:3181-SendThread(zkAAA.co.jp:2181)" #45 daemon prio=5 os_prio=0 tid=0x00007f0468010d80 nid=0x3ca9 runnable [0x00007f04779f8000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000e04f9270> (a sun.nio.ch.Util$3)
- locked <0x00000000e04f9260> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000e04f8dd0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
"ZkLedgerManagerScheduler-5-1" #44 prio=5 os_prio=0 tid=0x00007f04a4015d50 nid=0x2457 waiting on condition [0x00007f0477af9000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e020d640> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"ZkLedgerManagerScheduler-16-1" #43 prio=5 os_prio=0 tid=0x00007f04c80eab20 nid=0x2456 waiting on condition [0x00007f0477bfa000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e014bc50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"ZkLedgerManagerScheduler-15-1" #42 prio=5 os_prio=0 tid=0x00007f04c80046e0 nid=0x2455 waiting on condition [0x00007f0477cfb000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e00180e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"AuditorBookie-172.17.206.8:3181" #40 daemon prio=5 os_prio=0 tid=0x00007f049c117830 nid=0x5da4 waiting on condition [0x00007f0477dfc000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e04e54f8> (a java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
at org.apache.bookkeeper.replication.Auditor.checkAllLedgers(Auditor.java:696)
at org.apache.bookkeeper.replication.Auditor$5.run(Auditor.java:359)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-7-0" #38 prio=5 os_prio=0 tid=0x00007f049c00f690 nid=0x5da3 waiting on condition [0x00007f0477efd000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e002a850> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-6-0" #37 prio=5 os_prio=0 tid=0x00007f049c00dd90 nid=0x5da2 waiting on condition [0x00007f0477ffe000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e002af38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-5-0" #36 prio=5 os_prio=0 tid=0x00007f049c00c490 nid=0x5da1 waiting on condition [0x00007f04ec1b6000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e002b570> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-4-0" #35 prio=5 os_prio=0 tid=0x00007f049c00ab90 nid=0x5da0 waiting on condition [0x00007f04ec2b7000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e002bba8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-3-0" #34 prio=5 os_prio=0 tid=0x00007f049c009290 nid=0x5d9f waiting on condition [0x00007f04ec3b8000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e002c1e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-2-0" #33 prio=5 os_prio=0 tid=0x00007f049c007d80 nid=0x5d9e waiting on condition [0x00007f04ec4b9000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e0038670> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-1-0" #32 prio=5 os_prio=0 tid=0x00007f049c0068d0 nid=0x5d9d waiting on condition [0x00007f04ec5ba000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e0038ca8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-0-0" #31 prio=5 os_prio=0 tid=0x00007f049c003fb0 nid=0x5d9c waiting on condition [0x00007f04ec6bb000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e00392e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientScheduler-OrderedScheduler-0-0" #30 prio=5 os_prio=0 tid=0x00007f049c002bf0 nid=0x5d9b waiting on condition [0x00007f04ec7bc000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e0039918> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"AutoRecoveryDeathWatcher-3181" #27 prio=5 os_prio=0 tid=0x00007f05388bab40 nid=0x5be2 waiting on condition [0x00007f04ecabd000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at org.apache.bookkeeper.replication.AutoRecoveryMain$AutoRecoveryDeathWatcher.run(AutoRecoveryMain.java:212)
"ReplicationWorker" #25 prio=5 os_prio=0 tid=0x00007f05388b9650 nid=0x5be1 waiting on condition [0x00007f04ecbbe000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e1aa97d0> (a java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
at org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager.getLedgerToRereplicate(ZkLedgerUnderreplicationManager.java:538)
at org.apache.bookkeeper.replication.ReplicationWorker.rereplicate(ReplicationWorker.java:202)
at org.apache.bookkeeper.replication.ReplicationWorker.run(ReplicationWorker.java:168)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"AuditorElector-172.17.206.8:3181" #28 prio=5 os_prio=0 tid=0x00007f05388ad500 nid=0x5be0 waiting on condition [0x00007f04eccbf000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e00d76d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
"PendingReplicationTimer" #26 prio=5 os_prio=0 tid=0x00007f05388abfd0 nid=0x5bdf in Object.wait() [0x00007f04ecdc0000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
- locked <0x00000000e0133de0> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"main-EventThread" #23 daemon prio=5 os_prio=0 tid=0x00007f0538657050 nid=0x5bdd waiting on condition [0x00007f04ed2cf000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e0133f88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
"main-SendThread(zkAAA.co.jp:2181)" #22 daemon prio=5 os_prio=0 tid=0x00007f0538656090 nid=0x5bdc runnable [0x00007f04ed3d0000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000e01341e8> (a sun.nio.ch.Util$3)
- locked <0x00000000e01341d8> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000e01341f8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
"BookKeeperClientWorker-OrderedExecutor-7-0" #21 prio=5 os_prio=0 tid=0x00007f0538643cf0 nid=0x5bdb waiting on condition [0x00007f04ed6d1000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e00ef9c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-6-0" #20 prio=5 os_prio=0 tid=0x00007f05386427e0 nid=0x5bda waiting on condition [0x00007f04ed7d2000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e0102088> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-5-0" #19 prio=5 os_prio=0 tid=0x00007f05386412b0 nid=0x5bd9 waiting on condition [0x00007f04ed8d3000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e01026d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-4-0" #18 prio=5 os_prio=0 tid=0x00007f053863fda0 nid=0x5bd8 waiting on condition [0x00007f04ed9d4000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e0102d28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-3-0" #17 prio=5 os_prio=0 tid=0x00007f053863e870 nid=0x5bd7 waiting on condition [0x00007f04edad5000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e0103378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-2-0" #16 prio=5 os_prio=0 tid=0x00007f053863d360 nid=0x5bd6 waiting on condition [0x00007f05201ec000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e01039c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-1-0" #15 prio=5 os_prio=0 tid=0x00007f053863be50 nid=0x5bd5 waiting on condition [0x00007f05202ed000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e0113dc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientWorker-OrderedExecutor-0-0" #14 prio=5 os_prio=0 tid=0x00007f053863a9e0 nid=0x5bd4 waiting on condition [0x00007f05203ee000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e0114410> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"BookKeeperClientScheduler-OrderedScheduler-0-0" #13 prio=5 os_prio=0 tid=0x00007f05386389c0 nid=0x5bd3 waiting on condition [0x00007f05204ef000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e0134bc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
"main-EventThread" #11 daemon prio=5 os_prio=0 tid=0x00007f05385d3aa0 nid=0x5bd2 waiting on condition [0x00007f05207f0000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e1374598> (a java.util.concurrent.CompletableFuture$Signaller)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1693)
at java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3323)
at java.util.concurrent.CompletableFuture.waitingGet(CompletableFuture.java:1729)
at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1895)
at org.apache.bookkeeper.client.SyncCallbackUtils.waitForResult(SyncCallbackUtils.java:45)
at org.apache.bookkeeper.client.BookKeeperAdmin.openLedgerNoRecovery(BookKeeperAdmin.java:327)
at org.apache.bookkeeper.replication.Auditor$6.process(Auditor.java:645)
at org.apache.bookkeeper.replication.Auditor$6.process(Auditor.java:627)
at org.apache.bookkeeper.meta.AbstractZkLedgerManager$5.operationComplete(AbstractZkLedgerManager.java:510)
at org.apache.bookkeeper.meta.AbstractZkLedgerManager$5.operationComplete(AbstractZkLedgerManager.java:484)
at org.apache.bookkeeper.util.ZkUtils$6$1.processResult(ZkUtils.java:285)
at org.apache.bookkeeper.zookeeper.ZooKeeperClient$32$1.processResult(ZooKeeperClient.java:1395)
at org.apache.bookkeeper.zookeeper.ZooKeeperClient$31$1.processResult(ZooKeeperClient.java:1356)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:589)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
"main-SendThread(zkBBB.co.jp:2181)" #10 daemon prio=5 os_prio=0 tid=0x00007f05385b5310 nid=0x5bd1 runnable [0x00007f05208f1000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000e0135338> (a sun.nio.ch.Util$3)
- locked <0x00000000e0135328> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000e0135348> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
"Service Thread" #9 daemon prio=9 os_prio=0 tid=0x00007f05380e0780 nid=0x5bcf runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C1 CompilerThread3" #8 daemon prio=9 os_prio=0 tid=0x00007f05380d3c80 nid=0x5bce waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread2" #7 daemon prio=9 os_prio=0 tid=0x00007f05380d20a0 nid=0x5bcd waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread1" #6 daemon prio=9 os_prio=0 tid=0x00007f053801d230 nid=0x5bcc waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" #5 daemon prio=9 os_prio=0 tid=0x00007f05380c3ba0 nid=0x5bcb waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x00007f05380c2230 nid=0x5bca runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f0538098be0 nid=0x5bc9 in Object.wait() [0x00007f0521b3d000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
- locked <0x00000000e0107c40> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:212)
"Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f05380949d0 nid=0x5bc8 in Object.wait() [0x00007f0521c3e000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.lang.ref.Reference.tryHandlePending(Reference.java:191)
- locked <0x00000000e0107bd0> (a java.lang.ref.Reference$Lock)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)
"main" #1 prio=5 os_prio=0 tid=0x00007f053800b740 nid=0x5bbe in Object.wait() [0x00007f05402a6000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000e01338b8> (a org.apache.bookkeeper.replication.AutoRecoveryMain$AutoRecoveryDeathWatcher)
at java.lang.Thread.join(Thread.java:1252)
- locked <0x00000000e01338b8> (a org.apache.bookkeeper.replication.AutoRecoveryMain$AutoRecoveryDeathWatcher)
at java.lang.Thread.join(Thread.java:1326)
at org.apache.bookkeeper.replication.AutoRecoveryMain.join(AutoRecoveryMain.java:138)
at org.apache.bookkeeper.replication.AutoRecoveryMain.main(AutoRecoveryMain.java:326)
"VM Thread" os_prio=0 tid=0x00007f053808b550 nid=0x5bc7 runnable
"GC task thread#0 (ParallelGC)" os_prio=0 tid=0x00007f0538020140 nid=0x5bbf runnable
"GC task thread#1 (ParallelGC)" os_prio=0 tid=0x00007f05380216d0 nid=0x5bc0 runnable
"GC task thread#2 (ParallelGC)" os_prio=0 tid=0x00007f0538022c60 nid=0x5bc1 runnable
"GC task thread#3 (ParallelGC)" os_prio=0 tid=0x00007f05380241f0 nid=0x5bc2 runnable
"GC task thread#4 (ParallelGC)" os_prio=0 tid=0x00007f0538025780 nid=0x5bc3 runnable
"GC task thread#5 (ParallelGC)" os_prio=0 tid=0x00007f0538026d10 nid=0x5bc4 runnable
"GC task thread#6 (ParallelGC)" os_prio=0 tid=0x00007f05380282a0 nid=0x5bc5 runnable
"GC task thread#7 (ParallelGC)" os_prio=0 tid=0x00007f0538029830 nid=0x5bc6 runnable
"VM Periodic Task Thread" os_prio=0 tid=0x00007f05380e2590 nid=0x5bd0 waiting on condition
JNI global references: 290
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment