Created
October 24, 2011 19:41
-
-
Save alinpopa/1309937 to your computer and use it in GitHub Desktop.
Elasticsearch client hanging (too many open files)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2011-10-24 22:26:15 | |
| Full thread dump Java HotSpot(TM) 64-Bit Server VM (20.1-b02-383 mixed mode): | |
| "elasticsearch[cached]-pool-1-thread-1" daemon prio=5 tid=7fe3a51e4800 nid=0x1175cc000 waiting for monitor entry [1175cb000] | |
| java.lang.Thread.State: BLOCKED (on object monitor) | |
| at org.elasticsearch.client.transport.TransportClientNodesService$SimpleNodeSampler.sample(TransportClientNodesService.java:253) | |
| - waiting to lock <7f40c0048> (a org.elasticsearch.client.transport.TransportClientNodesService$SimpleNodeSampler) | |
| at org.elasticsearch.client.transport.TransportClientNodesService$ScheduledNodeSampler.run(TransportClientNodesService.java:243) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) | |
| at java.lang.Thread.run(Thread.java:680) | |
| "New I/O client worker #1-4" daemon prio=5 tid=7fe3ab9ed800 nid=0x1174c9000 runnable [1174c8000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) | |
| at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136) | |
| at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) | |
| - locked <7f300d450> (a sun.nio.ch.Util$2) | |
| - locked <7f300d440> (a java.util.Collections$UnmodifiableSet) | |
| - locked <7f300d238> (a sun.nio.ch.KQueueSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) | |
| at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:38) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:163) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:44) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) | |
| at java.lang.Thread.run(Thread.java:680) | |
| "New I/O client worker #1-1" daemon prio=5 tid=7fe3ab9eb000 nid=0x1173c6000 runnable [1173c5000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) | |
| at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136) | |
| at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) | |
| - locked <7f300ca00> (a sun.nio.ch.Util$2) | |
| - locked <7f300c9f0> (a java.util.Collections$UnmodifiableSet) | |
| - locked <7f300c7e8> (a sun.nio.ch.KQueueSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) | |
| at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:38) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:163) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:44) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) | |
| at java.lang.Thread.run(Thread.java:680) | |
| "New I/O client worker #1-2" daemon prio=5 tid=7fe3ab9e9000 nid=0x1172c3000 runnable [1172c2000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) | |
| at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136) | |
| at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) | |
| - locked <7f300bfb0> (a sun.nio.ch.Util$2) | |
| - locked <7f300bfa0> (a java.util.Collections$UnmodifiableSet) | |
| - locked <7f300bd98> (a sun.nio.ch.KQueueSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) | |
| at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:38) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:163) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:44) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) | |
| at java.lang.Thread.run(Thread.java:680) | |
| "New I/O client worker #1-6" daemon prio=5 tid=7fe3aba52000 nid=0x1171c0000 runnable [1171bf000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) | |
| at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136) | |
| at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) | |
| - locked <7f300b510> (a sun.nio.ch.Util$2) | |
| - locked <7f300b500> (a java.util.Collections$UnmodifiableSet) | |
| - locked <7f300b2f8> (a sun.nio.ch.KQueueSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) | |
| at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:38) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:163) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:44) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) | |
| at java.lang.Thread.run(Thread.java:680) | |
| "New I/O client worker #1-3" daemon prio=5 tid=7fe3aba4f800 nid=0x1170bd000 runnable [1170bc000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) | |
| at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136) | |
| at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) | |
| - locked <7f300aac0> (a sun.nio.ch.Util$2) | |
| - locked <7f300aab0> (a java.util.Collections$UnmodifiableSet) | |
| - locked <7f300a8a8> (a sun.nio.ch.KQueueSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) | |
| at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:38) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:163) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:44) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) | |
| at java.lang.Thread.run(Thread.java:680) | |
| "New I/O client worker #1-5" daemon prio=5 tid=7fe3aba4d800 nid=0x116fba000 runnable [116fb9000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) | |
| at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136) | |
| at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) | |
| - locked <7f300a070> (a sun.nio.ch.Util$2) | |
| - locked <7f300a060> (a java.util.Collections$UnmodifiableSet) | |
| - locked <7f3009e58> (a sun.nio.ch.KQueueSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) | |
| at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:38) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:163) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:44) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) | |
| at java.lang.Thread.run(Thread.java:680) | |
| "New I/O client worker #1-7" daemon prio=5 tid=7fe3aba4b000 nid=0x116eb7000 runnable [116eb6000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) | |
| at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136) | |
| at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) | |
| - locked <7f3009620> (a sun.nio.ch.Util$2) | |
| - locked <7f3009610> (a java.util.Collections$UnmodifiableSet) | |
| - locked <7f3009408> (a sun.nio.ch.KQueueSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) | |
| at org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:38) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:163) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:44) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) | |
| at java.lang.Thread.run(Thread.java:680) | |
| "elasticsearch[Black Mamba]transport_client_boss-pool-9-thread-1" daemon prio=5 tid=7fe3aba37800 nid=0x116db4000 waiting on condition [116db3000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at sun.misc.Unsafe.park(Native Method) | |
| - parking to wait for <7f411fcd0> (a java.util.concurrent.SynchronousQueue$TransferStack) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) | |
| at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:424) | |
| at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:323) | |
| at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:874) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:945) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) | |
| at java.lang.Thread.run(Thread.java:680) | |
| "elasticsearch[Black Mamba][scheduler]-pool-8-thread-1" daemon prio=5 tid=7fe3ac0cf000 nid=0x116c50000 waiting on condition [116c4f000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at sun.misc.Unsafe.park(Native Method) | |
| - parking to wait for <7f40afc50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) | |
| at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) | |
| at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) | |
| at java.util.concurrent.DelayQueue.take(DelayQueue.java:160) | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:609) | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:602) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) | |
| at java.lang.Thread.run(Thread.java:680) | |
| "elasticsearch[Black Mamba][timer]" daemon prio=5 tid=7fe3ac06e000 nid=0x116b4d000 waiting on condition [116b4c000] | |
| java.lang.Thread.State: TIMED_WAITING (sleeping) | |
| at java.lang.Thread.sleep(Native Method) | |
| at org.elasticsearch.threadpool.ThreadPool$EstimatedTimeThread.run(ThreadPool.java:285) | |
| "Monitor Ctrl-Break" daemon prio=5 tid=7fe3ab8a1000 nid=0x116857000 runnable [116856000] | |
| java.lang.Thread.State: RUNNABLE | |
| at java.net.PlainSocketImpl.socketAccept(Native Method) | |
| at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:408) | |
| - locked <7f40bf840> (a java.net.SocksSocketImpl) | |
| at java.net.ServerSocket.implAccept(ServerSocket.java:462) | |
| at java.net.ServerSocket.accept(ServerSocket.java:430) | |
| at com.intellij.rt.execution.application.AppMain$1.run(AppMain.java:82) | |
| at java.lang.Thread.run(Thread.java:680) | |
| "Low Memory Detector" daemon prio=5 tid=7fe3a612a800 nid=0x11627e000 runnable [00000000] | |
| java.lang.Thread.State: RUNNABLE | |
| "C2 CompilerThread1" daemon prio=9 tid=7fe3a612a000 nid=0x11617b000 waiting on condition [00000000] | |
| java.lang.Thread.State: RUNNABLE | |
| "C2 CompilerThread0" daemon prio=9 tid=7fe3a6129000 nid=0x116078000 waiting on condition [00000000] | |
| java.lang.Thread.State: RUNNABLE | |
| "Signal Dispatcher" daemon prio=9 tid=7fe3a6128800 nid=0x115f75000 waiting on condition [00000000] | |
| java.lang.Thread.State: RUNNABLE | |
| "Surrogate Locker Thread (Concurrent GC)" daemon prio=5 tid=7fe3a6127800 nid=0x115e72000 waiting on condition [00000000] | |
| java.lang.Thread.State: RUNNABLE | |
| "Finalizer" daemon prio=8 tid=7fe3a6123000 nid=0x115bae000 in Object.wait() [115bad000] | |
| java.lang.Thread.State: WAITING (on object monitor) | |
| at java.lang.Object.wait(Native Method) | |
| - waiting on <7f40bfc18> (a java.lang.ref.ReferenceQueue$Lock) | |
| at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118) | |
| - locked <7f40bfc18> (a java.lang.ref.ReferenceQueue$Lock) | |
| at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134) | |
| at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) | |
| "Reference Handler" daemon prio=10 tid=7fe3a6122000 nid=0x115aab000 in Object.wait() [115aaa000] | |
| java.lang.Thread.State: WAITING (on object monitor) | |
| at java.lang.Object.wait(Native Method) | |
| - waiting on <7f40a0100> (a java.lang.ref.Reference$Lock) | |
| at java.lang.Object.wait(Object.java:485) | |
| at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) | |
| - locked <7f40a0100> (a java.lang.ref.Reference$Lock) | |
| "main" prio=5 tid=7fe3a6000800 nid=0x10dd35000 waiting on condition [10dd33000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at sun.misc.Unsafe.park(Native Method) | |
| - parking to wait for <7f31ea4f0> (a org.elasticsearch.common.util.concurrent.AbstractFuture$Sync) | |
| at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) | |
| at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:811) | |
| at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:969) | |
| at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1281) | |
| at org.elasticsearch.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:249) | |
| at org.elasticsearch.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:78) | |
| at org.elasticsearch.transport.PlainTransportFuture.txGet(PlainTransportFuture.java:45) | |
| at org.elasticsearch.transport.PlainTransportFuture.txGet(PlainTransportFuture.java:35) | |
| at org.elasticsearch.client.transport.TransportClientNodesService$SimpleNodeSampler.sample(TransportClientNodesService.java:267) | |
| - locked <7f40c0048> (a org.elasticsearch.client.transport.TransportClientNodesService$SimpleNodeSampler) | |
| at org.elasticsearch.client.transport.TransportClientNodesService.addTransportAddress(TransportClientNodesService.java:124) | |
| at org.elasticsearch.client.transport.TransportClient.addTransportAddress(TransportClient.java:200) | |
| at org.my.test.search.ElasticSearchClient.<init>(ElasticSearchClient.scala:31) | |
| at org.my.test.search.elasticsearch.ElasticSearchClient$.main(ElasticSearchClient.scala:107) | |
| at org.my.test.search.elasticsearch.ElasticSearchClient.main(ElasticSearchClient.scala) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) | |
| at java.lang.reflect.Method.invoke(Method.java:597) | |
| at com.intellij.rt.execution.application.AppMain.main(AppMain.java:120) | |
| "VM Thread" prio=9 tid=7fe3a611d800 nid=0x1159a8000 runnable | |
| "Gang worker#0 (Parallel GC Threads)" prio=9 tid=7fe3a6002000 nid=0x11106e000 runnable | |
| "Gang worker#1 (Parallel GC Threads)" prio=9 tid=7fe3a6002800 nid=0x111171000 runnable | |
| "Gang worker#2 (Parallel GC Threads)" prio=9 tid=7fe3a6003000 nid=0x111274000 runnable | |
| "Gang worker#3 (Parallel GC Threads)" prio=9 tid=7fe3a6004000 nid=0x111377000 runnable | |
| "Gang worker#4 (Parallel GC Threads)" prio=9 tid=7fe3a6004800 nid=0x11147a000 runnable | |
| "Gang worker#5 (Parallel GC Threads)" prio=9 tid=7fe3a6005000 nid=0x11157d000 runnable | |
| "Gang worker#6 (Parallel GC Threads)" prio=9 tid=7fe3a6005800 nid=0x111680000 runnable | |
| "Gang worker#7 (Parallel GC Threads)" prio=9 tid=7fe3a6006800 nid=0x111783000 runnable | |
| "Concurrent Mark-Sweep GC Thread" prio=9 tid=7fe3a60e3800 nid=0x11564e000 runnable | |
| "Gang worker#0 (Parallel CMS Threads)" prio=9 tid=7fe3a60e2800 nid=0x114c48000 runnable | |
| "Gang worker#1 (Parallel CMS Threads)" prio=9 tid=7fe3a60e3000 nid=0x114d4b000 runnable | |
| "VM Periodic Task Thread" prio=10 tid=7fe3a613c800 nid=0x116381000 waiting on condition | |
| "Exception Catcher Thread" prio=10 tid=7fe3a6001800 nid=0x10de97000 runnable | |
| JNI global references: 1216 | |
| Heap | |
| par new generation total 19136K, used 5188K [7f3000000, 7f44c0000, 7f44c0000) | |
| eden space 17024K, 22% used [7f3000000, 7f33cb928, 7f40a0000) | |
| from space 2112K, 61% used [7f40a0000, 7f41e5a80, 7f42b0000) | |
| to space 2112K, 0% used [7f42b0000, 7f42b0000, 7f44c0000) | |
| concurrent mark-sweep generation total 63872K, used 0K [7f44c0000, 7f8320000, 7fae00000) | |
| concurrent-mark-sweep perm gen total 21248K, used 12223K [7fae00000, 7fc2c0000, 800000000) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment