Created
December 18, 2013 15:23
-
-
Save jprante/8024139 to your computer and use it in GitHub Desktop.
OOM of ES 1.0.0.Beta2 cluster
3 RHEL 6.3 nodes (zephyros, boreas, notos) Java 1.8.0-ea, JVM 25.0-b56, 4GB heap, G1 GC
Workload: mixed term queries with bulk indexing (5000 docs per second)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [2013-12-17 13:06:30,177][INFO ][node ] [Collector] version[1.0.0.Beta2], pid[30239], build[296cfbe/2013-12-02T15:46:27Z] | |
| [2013-12-17 13:06:30,177][INFO ][node ] [Collector] initializing ... | |
| [2013-12-17 13:06:30,196][INFO ][plugins ] [Collector] loaded [analysis-german, ingest], sites [] | |
| [2013-12-17 13:06:33,051][INFO ][node ] [Collector] initialized | |
| [2013-12-17 13:06:33,052][INFO ][node ] [Collector] starting ... | |
| [2013-12-17 13:06:34,055][INFO ][transport ] [Collector] bound_address {inet[/10.3.2.32:19300]}, publish_address {inet[/10.3.2.32:19300]} | |
| [2013-12-17 13:06:37,172][INFO ][cluster.service ] [Collector] detected_master [Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]], added {[Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]],}, reason: zen-disco-receive(from master [[Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]]]) | |
| [2013-12-17 13:06:37,194][INFO ][discovery ] [Collector] zbn-1.0/_vXVmJi6RqCmMiS3AWXt6Q | |
| [2013-12-17 13:06:37,240][INFO ][http ] [Collector] bound_address {inet[/10.3.2.32:19200]}, publish_address {inet[/10.3.2.32:19200]} | |
| [2013-12-17 13:06:37,241][INFO ][node ] [Collector] started | |
| [2013-12-17 13:06:42,196][INFO ][cluster.service ] [Collector] added {[The Profile][ldtEHWFxRcaZBHQaKFS_TQ][inet[/10.3.2.33:19300]],}, reason: zen-disco-receive(from master [[Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]]]) | |
| [2013-12-17 13:11:55,506][INFO ][index.shard.service ] [Collector] [fix][2] updating refresh_interval from [1s] to [-1] | |
| [2013-12-17 13:11:55,507][INFO ][index.shard.service ] [Collector] [fix][3] updating refresh_interval from [1s] to [-1] | |
| [2013-12-17 13:11:55,507][INFO ][index.shard.service ] [Collector] [fix][1] updating refresh_interval from [1s] to [-1] | |
| [2013-12-17 13:26:07,002][DEBUG][action.search.type ] [Collector] [501346] Failed to execute query phase | |
| org.elasticsearch.search.SearchContextMissingException: No search context found for id [501346] | |
| at org.elasticsearch.search.SearchService.findContext(SearchService.java:446) | |
| at org.elasticsearch.search.SearchService.executeScan(SearchService.java:197) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:474) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.executePhase(TransportSearchScrollScanAction.java:210) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction$2.run(TransportSearchScrollScanAction.java:180) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| [2013-12-17 13:26:07,009][DEBUG][action.search.type ] [Collector] [355131] Failed to execute query phase | |
| org.elasticsearch.transport.RemoteTransportException: [Authority][inet[/10.3.2.31:19300]][search/phase/scan/scroll] | |
| Caused by: org.elasticsearch.search.SearchContextMissingException: No search context found for id [355131] | |
| at org.elasticsearch.search.SearchService.findContext(SearchService.java:446) | |
| at org.elasticsearch.search.SearchService.executeScan(SearchService.java:197) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction$SearchScanScrollTransportHandler.messageReceived(SearchServiceTransportAction.java:791) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction$SearchScanScrollTransportHandler.messageReceived(SearchServiceTransportAction.java:780) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:270) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| [2013-12-17 13:38:22,727][DEBUG][action.search.type ] [Collector] [1018124] Failed to execute query phase | |
| org.elasticsearch.search.SearchContextMissingException: No search context found for id [1018124] | |
| at org.elasticsearch.search.SearchService.findContext(SearchService.java:446) | |
| at org.elasticsearch.search.SearchService.executeScan(SearchService.java:197) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:474) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.executePhase(TransportSearchScrollScanAction.java:210) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction$2.run(TransportSearchScrollScanAction.java:180) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| [2013-12-17 13:38:22,729][DEBUG][action.search.type ] [Collector] [716562] Failed to execute query phase | |
| org.elasticsearch.transport.RemoteTransportException: [Authority][inet[/10.3.2.31:19300]][search/phase/scan/scroll] | |
| Caused by: org.elasticsearch.search.SearchContextMissingException: No search context found for id [716562] | |
| at org.elasticsearch.search.SearchService.findContext(SearchService.java:446) | |
| at org.elasticsearch.search.SearchService.executeScan(SearchService.java:197) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction$SearchScanScrollTransportHandler.messageReceived(SearchServiceTransportAction.java:791) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction$SearchScanScrollTransportHandler.messageReceived(SearchServiceTransportAction.java:780) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:270) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| [2013-12-17 14:35:20,549][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:35:20,549][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:35:20,550][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:35:20,550][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:35:20,549][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:35:20,550][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:35:20,550][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:35:20,550][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:35:20,549][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:36:58,587][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:36:58,588][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:36:58,592][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:36:58,589][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:36:58,589][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:36:58,589][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:36:58,588][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:36:58,677][WARN ][index.merge.scheduler ] [Collector] [fix][2] failed to merge | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:36:58,678][WARN ][index.engine.robin ] [Collector] [fix][2] failed engine | |
| org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: Java heap space | |
| at org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider$CustomConcurrentMergeScheduler.handleMergeException(ConcurrentMergeSchedulerProvider.java:109) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:518) | |
| Caused by: java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:36:58,694][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[STARTED], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [engine failure, message [MergeException[java.lang.OutOfMemoryError: Java heap space]; nested: OutOfMemoryError[Java heap space]; ]] | |
| [2013-12-17 14:36:59,598][WARN ][transport ] [Collector] Received response for a request that has timed out, sent [97111ms] ago, timed out [1011ms] ago, action [discovery/zen/fd/masterPing], node [[Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]]], id [4167245] | |
| [2013-12-17 14:36:59,601][WARN ][indices.cluster ] [Collector] [fix][2] master [[Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]]] marked shard as started, but shard has not been created, mark shard as failed | |
| [2013-12-17 14:36:59,602][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[STARTED], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [master [Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]] marked shard as started, but shard has not been created, mark shard as failed] | |
| [2013-12-17 14:38:36,845][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:38:36,846][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:38:36,850][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:39:08,799][INFO ][cluster.service ] [Collector] removed {[The Profile][ldtEHWFxRcaZBHQaKFS_TQ][inet[/10.3.2.33:19300]],}, reason: zen-disco-receive(from master [[Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]]]) | |
| [2013-12-17 14:39:12,480][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:39:12,489][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:39:12,498][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:39:19,041][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:39:19,041][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:39:19,046][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:39:25,682][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:39:25,688][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:39:25,693][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:40:02,439][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:40:02,440][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:40:02,445][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:40:38,891][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:40:38,891][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:40:38,895][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:40:45,445][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:40:45,445][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:40:45,450][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:40:51,911][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:40:51,912][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:40:51,916][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:41:28,547][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:41:28,547][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:41:28,552][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:41:35,482][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:41:35,483][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:41:35,487][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:42:12,006][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:42:12,006][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:42:12,011][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:42:18,544][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:42:18,544][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:42:18,549][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:42:25,088][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:42:25,088][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:42:25,093][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:43:01,649][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:43:01,649][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:43:01,652][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:43:37,900][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:43:37,901][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:43:37,904][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:43:44,574][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:43:44,574][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:43:44,578][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:43:50,996][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:43:50,997][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:43:51,001][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:44:27,441][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:44:27,442][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:44:27,450][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:44:34,131][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:44:34,131][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:44:34,135][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:45:10,843][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:45:10,843][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:45:10,865][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:45:17,169][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:45:17,169][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:45:17,174][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:45:23,941][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:45:23,941][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:45:23,945][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:45:40,536][DEBUG][action.search.type ] [Collector] Node [ldtEHWFxRcaZBHQaKFS_TQ] not available for scroll request [c2NhbjszOzM0NTk1NzI6bGR0RUhXRnhSY2FaQkhRYUtGU19UUTszMzYzNjg3Ol92WFZtSmk2UnFDbU1pUzNBV1h0NlE7MzQ1OTU3MTpsZHRFSFdGeFJjYVpCSFFhS0ZTX1RROzE7dG90YWxfaGl0czo2OTs=] | |
| [2013-12-17 14:45:40,542][DEBUG][action.search.type ] [Collector] Node [ldtEHWFxRcaZBHQaKFS_TQ] not available for scroll request [c2NhbjszOzM0NTk1NzI6bGR0RUhXRnhSY2FaQkhRYUtGU19UUTszMzYzNjg3Ol92WFZtSmk2UnFDbU1pUzNBV1h0NlE7MzQ1OTU3MTpsZHRFSFdGeFJjYVpCSFFhS0ZTX1RROzE7dG90YWxfaGl0czo2OTs=] | |
| [2013-12-17 14:45:40,542][DEBUG][action.search.type ] [Collector] Node [ldtEHWFxRcaZBHQaKFS_TQ] not available for scroll request [c2NhbjszOzM0NTk1NzI6bGR0RUhXRnhSY2FaQkhRYUtGU19UUTszMzYzNjg3Ol92WFZtSmk2UnFDbU1pUzNBV1h0NlE7MzQ1OTU3MTpsZHRFSFdGeFJjYVpCSFFhS0ZTX1RROzE7dG90YWxfaGl0czo2OTs=] | |
| [2013-12-17 14:45:40,542][DEBUG][action.search.type ] [Collector] Node [ldtEHWFxRcaZBHQaKFS_TQ] not available for scroll request [c2NhbjszOzM0NTk1NzI6bGR0RUhXRnhSY2FaQkhRYUtGU19UUTszMzYzNjg3Ol92WFZtSmk2UnFDbU1pUzNBV1h0NlE7MzQ1OTU3MTpsZHRFSFdGeFJjYVpCSFFhS0ZTX1RROzE7dG90YWxfaGl0czo2OTs=] | |
| [2013-12-17 14:45:40,600][WARN ][cluster.action.shard ] [Collector] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[STARTED], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [engine failure, message [MergeException[java.lang.OutOfMemoryError: Java heap space]; nested: OutOfMemoryError[Java heap space]; ]] | |
| [2013-12-17 14:45:41,576][DEBUG][action.search.type ] [Collector] Node [ldtEHWFxRcaZBHQaKFS_TQ] not available for scroll request [c2NhbjsyOzMzNjM2ODg6X3ZYVm1KaTZScUNtTWlTM0FXWHQ2UTszNDU5NTc0OmxkdEVIV0Z4UmNhWkJIUWFLRlNfVFE7MTt0b3RhbF9oaXRzOjE7] | |
| [2013-12-17 14:45:41,576][DEBUG][action.search.type ] [Collector] Node [ldtEHWFxRcaZBHQaKFS_TQ] not available for scroll request [c2NhbjsyOzMzNjM2ODg6X3ZYVm1KaTZScUNtTWlTM0FXWHQ2UTszNDU5NTc0OmxkdEVIV0Z4UmNhWkJIUWFLRlNfVFE7MTt0b3RhbF9oaXRzOjE7] | |
| [2013-12-17 14:46:30,332][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:46:30,332][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:46:30,336][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:46:36,937][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:46:36,937][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:46:36,941][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:46:43,138][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:46:43,138][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:46:43,142][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:47:19,730][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:47:19,730][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:47:19,734][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:47:26,355][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:47:26,355][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:47:26,360][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:48:02,804][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:48:02,805][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:48:02,809][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:48:09,477][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:48:09,477][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:48:09,481][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:48:15,933][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:48:15,933][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:48:15,936][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:48:52,548][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:48:52,548][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:48:52,552][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:48:58,851][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:48:58,851][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:48:58,855][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:49:35,247][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:49:35,247][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:49:35,251][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:49:41,814][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:49:41,814][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:49:41,818][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:49:48,207][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:49:48,207][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:49:48,211][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:50:24,619][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:50:24,619][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:50:24,622][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:50:31,421][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:50:31,421][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:50:31,425][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:51:07,972][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:51:07,972][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:51:07,975][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:51:14,356][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:51:14,356][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:51:14,364][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:51:21,063][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:51:21,064][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:51:21,067][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:51:57,561][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:51:57,561][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:51:57,564][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:52:03,750][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:52:03,751][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:52:03,774][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:52:40,319][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:52:40,319][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:52:40,323][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:52:46,900][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:52:46,901][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:52:46,905][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:52:53,282][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:52:53,282][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:52:53,286][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:53:29,905][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:53:29,906][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:53:29,910][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:53:36,469][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:53:36,470][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:53:36,473][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:54:13,009][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:54:13,010][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:54:13,013][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:54:19,473][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:54:19,473][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:54:19,477][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:54:23,106][WARN ][indices.cluster ] [Collector] [ezb][0] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [ezb][0]: Recovery failed from [Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]] into [Collector][_vXVmJi6RqCmMiS3AWXt6Q][inet[/10.3.2.32:19300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:303) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:65) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$3.run(RecoveryTarget.java:180) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: [Authority][inet[/10.3.2.31:19300]][index/shard/recovery/startRecovery] | |
| Caused by: org.elasticsearch.index.engine.RecoveryEngineException: [ezb][0] Phase[2] Execution failed | |
| at org.elasticsearch.index.engine.robin.RobinEngine.recover(RobinEngine.java:1103) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.recover(InternalIndexShard.java:629) | |
| at org.elasticsearch.indices.recovery.RecoverySource.recover(RecoverySource.java:117) | |
| at org.elasticsearch.indices.recovery.RecoverySource.access$1600(RecoverySource.java:61) | |
| at org.elasticsearch.indices.recovery.RecoverySource$StartRecoveryTransportRequestHandler.messageReceived(RecoverySource.java:333) | |
| at org.elasticsearch.indices.recovery.RecoverySource$StartRecoveryTransportRequestHandler.messageReceived(RecoverySource.java:319) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:270) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.transport.ReceiveTimeoutTransportException: [Collector][inet[/10.3.2.32:19300]][index/shard/recovery/prepareTranslog] request_id [4899165] timed out after [900001ms] | |
| at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:356) | |
| ... 3 more | |
| [2013-12-17 14:54:25,940][WARN ][cluster.action.shard ] [Collector] [ezb][0] sending failed shard for [ezb][0], node[_vXVmJi6RqCmMiS3AWXt6Q], [R], s[INITIALIZING], indexUUID [dPG0zTc_SuOcz3WH09MOQA], reason [Failed to start shard, message [RecoveryFailedException[[ezb][0]: Recovery failed from [Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]] into [Collector][_vXVmJi6RqCmMiS3AWXt6Q][inet[/10.3.2.32:19300]]]; nested: RemoteTransportException[[Authority][inet[/10.3.2.31:19300]][index/shard/recovery/startRecovery]]; nested: RecoveryEngineException[[ezb][0] Phase[2] Execution failed]; nested: ReceiveTimeoutTransportException[[Collector][inet[/10.3.2.32:19300]][index/shard/recovery/prepareTranslog] request_id [4899165] timed out after [900001ms]]; ]] | |
| [2013-12-17 14:54:25,949][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:54:25,955][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:54:25,958][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:55:02,503][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:55:02,504][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:55:02,509][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:55:08,713][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:55:08,713][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:55:08,717][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:55:15,313][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:55:15,313][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:55:15,317][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:55:51,954][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:55:51,954][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:55:51,958][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:55:58,273][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:55:58,274][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:55:58,278][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:56:04,805][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:56:04,805][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:56:04,808][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:56:41,405][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:56:41,405][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:56:41,409][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:56:47,908][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:56:47,908][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:56:47,912][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:57:24,297][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:57:24,297][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:57:24,307][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:57:30,662][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:57:30,663][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:57:30,666][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:57:37,343][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:57:37,343][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:57:37,347][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:58:13,833][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:58:13,833][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:58:13,861][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:58:20,390][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:58:20,390][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:58:20,394][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:58:56,792][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:58:56,793][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:58:56,796][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:59:03,294][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:59:03,294][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:59:03,297][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:59:09,991][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:59:09,991][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:59:09,996][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:59:46,680][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:59:46,680][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:59:46,684][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 14:59:53,034][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 14:59:53,034][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:59:53,038][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:00:29,310][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:00:29,311][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:00:29,315][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:00:35,611][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:00:35,611][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:00:35,616][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:00:42,132][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:00:42,132][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:00:42,136][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:01:18,354][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:01:18,354][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:01:18,358][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:01:24,960][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:01:24,960][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:01:24,964][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:02:01,871][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:02:01,872][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:02:01,876][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:02:08,358][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:02:08,358][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:02:08,366][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:02:44,971][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:02:44,971][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:02:44,974][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:02:51,369][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:02:51,369][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:02:51,372][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:02:57,942][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:02:57,943][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:02:57,946][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:03:34,361][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:03:34,361][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:03:34,365][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:03:40,800][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:03:40,800][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:03:40,804][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:04:17,571][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:04:17,571][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:04:17,580][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:04:24,295][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:04:24,295][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:04:24,299][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:04:30,780][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:04:30,781][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:04:30,805][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:05:07,349][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:05:07,349][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:05:07,353][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:05:13,680][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:05:13,680][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:05:13,683][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:05:50,078][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:05:50,078][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:05:50,081][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:05:56,703][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:05:56,704][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:05:56,707][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:06:03,149][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:06:03,149][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:06:03,153][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:06:39,765][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:06:39,765][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:06:39,769][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:06:46,447][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:06:46,447][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:06:46,452][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:07:22,912][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:07:22,912][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:07:22,916][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:07:29,399][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:07:29,399][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:07:29,403][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:07:35,570][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:07:35,570][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:07:35,574][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:08:12,164][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:08:12,164][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:08:12,169][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:08:18,842][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:08:18,843][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:08:18,846][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:08:55,186][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:08:55,186][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:08:55,190][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:09:01,820][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:09:01,820][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:09:01,824][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:09:08,436][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:09:08,436][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:09:08,440][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:09:44,961][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:09:44,961][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:09:44,965][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:09:51,154][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:09:51,155][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:09:51,158][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:10:02,529][WARN ][indices.cluster ] [Collector] [ezb][0] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [ezb][0]: Recovery failed from [Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]] into [Collector][_vXVmJi6RqCmMiS3AWXt6Q][inet[/10.3.2.32:19300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:303) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:65) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:171) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: [Authority][inet[/10.3.2.31:19300]][index/shard/recovery/startRecovery] | |
| Caused by: org.elasticsearch.index.engine.RecoveryEngineException: [ezb][0] Phase[2] Execution failed | |
| at org.elasticsearch.index.engine.robin.RobinEngine.recover(RobinEngine.java:1103) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.recover(InternalIndexShard.java:629) | |
| at org.elasticsearch.indices.recovery.RecoverySource.recover(RecoverySource.java:117) | |
| at org.elasticsearch.indices.recovery.RecoverySource.access$1600(RecoverySource.java:61) | |
| at org.elasticsearch.indices.recovery.RecoverySource$StartRecoveryTransportRequestHandler.messageReceived(RecoverySource.java:333) | |
| at org.elasticsearch.indices.recovery.RecoverySource$StartRecoveryTransportRequestHandler.messageReceived(RecoverySource.java:319) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:270) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.transport.ReceiveTimeoutTransportException: [Collector][inet[/10.3.2.32:19300]][index/shard/recovery/prepareTranslog] request_id [4900750] timed out after [900000ms] | |
| at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:356) | |
| ... 3 more | |
| [2013-12-17 15:10:02,536][WARN ][cluster.action.shard ] [Collector] [ezb][0] sending failed shard for [ezb][0], node[_vXVmJi6RqCmMiS3AWXt6Q], [R], s[INITIALIZING], indexUUID [dPG0zTc_SuOcz3WH09MOQA], reason [Failed to start shard, message [RecoveryFailedException[[ezb][0]: Recovery failed from [Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]] into [Collector][_vXVmJi6RqCmMiS3AWXt6Q][inet[/10.3.2.32:19300]]]; nested: RemoteTransportException[[Authority][inet[/10.3.2.31:19300]][index/shard/recovery/startRecovery]]; nested: RecoveryEngineException[[ezb][0] Phase[2] Execution failed]; nested: ReceiveTimeoutTransportException[[Collector][inet[/10.3.2.32:19300]][index/shard/recovery/prepareTranslog] request_id [4900750] timed out after [900000ms]]; ]] | |
| [2013-12-17 15:10:27,609][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:10:27,610][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:10:27,614][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:10:34,399][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:10:34,399][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:10:34,404][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:10:41,128][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:10:41,128][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:10:41,157][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:10:47,725][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:10:47,726][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:10:47,729][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:11:24,335][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:11:24,335][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:11:24,338][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:11:30,878][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:11:30,879][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:11:30,882][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:11:37,207][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:11:37,207][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:11:37,211][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:12:13,608][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:12:13,608][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:12:13,612][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:12:20,333][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:12:20,333][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:12:20,337][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:12:57,025][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:12:57,025][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:12:57,029][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:13:03,573][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:13:03,573][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:13:03,577][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:13:10,169][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:13:10,169][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:13:10,173][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:13:46,680][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:13:46,681][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:13:46,685][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:13:52,980][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:13:52,980][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:13:52,984][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:14:29,379][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:14:29,379][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:14:29,383][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:14:36,094][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:14:36,095][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:14:36,099][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:14:42,568][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:14:42,568][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:14:42,572][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:15:18,872][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:15:18,873][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:15:18,876][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:15:25,188][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:15:25,188][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:15:25,192][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:16:01,451][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:16:01,451][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:16:01,455][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:16:07,518][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:16:07,519][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:16:07,522][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:16:13,638][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:16:13,638][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:16:13,642][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:16:50,114][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:16:50,115][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:16:50,119][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:16:56,508][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:16:56,508][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:16:56,535][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:17:32,931][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:17:32,932][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:32,936][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:17:39,169][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:17:39,169][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:39,172][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:17:45,194][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:17:45,194][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:45,198][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:18:21,247][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:18:21,247][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:21,251][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:18:27,684][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:18:27,684][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:27,687][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:19:03,866][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:19:03,866][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:19:03,870][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:19:10,272][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:19:10,273][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:19:10,276][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:19:16,688][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:19:16,689][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:19:16,692][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:19:52,893][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:19:52,893][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:19:52,897][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:19:58,906][WARN ][index.engine.robin ] [Collector] [fix][2] shard is locked, releasing lock | |
| [2013-12-17 15:19:58,906][WARN ][indices.cluster ] [Collector] [fix][2] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][2] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][2] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:19:58,910][WARN ][cluster.action.shard ] [Collector] [fix][2] sending failed shard for [fix][2], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][2] failed recovery]; nested: EngineCreationFailureException[[fix][2] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/2/index/write.lock]; ]] | |
| [2013-12-17 15:25:34,441][WARN ][indices.cluster ] [Collector] [ezb][0] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [ezb][0]: Recovery failed from [Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]] into [Collector][_vXVmJi6RqCmMiS3AWXt6Q][inet[/10.3.2.32:19300]] | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:303) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:65) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:171) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: [Authority][inet[/10.3.2.31:19300]][index/shard/recovery/startRecovery] | |
| Caused by: org.elasticsearch.index.engine.RecoveryEngineException: [ezb][0] Phase[1] Execution failed | |
| at org.elasticsearch.index.engine.robin.RobinEngine.recover(RobinEngine.java:1079) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.recover(InternalIndexShard.java:629) | |
| at org.elasticsearch.indices.recovery.RecoverySource.recover(RecoverySource.java:117) | |
| at org.elasticsearch.indices.recovery.RecoverySource.access$1600(RecoverySource.java:61) | |
| at org.elasticsearch.indices.recovery.RecoverySource$StartRecoveryTransportRequestHandler.messageReceived(RecoverySource.java:333) | |
| at org.elasticsearch.indices.recovery.RecoverySource$StartRecoveryTransportRequestHandler.messageReceived(RecoverySource.java:319) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:270) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.indices.recovery.RecoverFilesRecoveryException: [ezb][0] Failed to transfer [1] files with total size of [76mb] | |
| at org.elasticsearch.indices.recovery.RecoverySource$1.phase1(RecoverySource.java:231) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.recover(RobinEngine.java:1072) | |
| ... 9 more | |
| Caused by: org.elasticsearch.transport.ReceiveTimeoutTransportException: [Collector][inet[/10.3.2.32:19300]][index/shard/recovery/cleanFiles] request_id [4901909] timed out after [900001ms] | |
| at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:356) | |
| ... 3 more | |
| [2013-12-17 15:25:34,448][WARN ][cluster.action.shard ] [Collector] [ezb][0] sending failed shard for [ezb][0], node[_vXVmJi6RqCmMiS3AWXt6Q], [R], s[INITIALIZING], indexUUID [dPG0zTc_SuOcz3WH09MOQA], reason [Failed to start shard, message [RecoveryFailedException[[ezb][0]: Recovery failed from [Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]] into [Collector][_vXVmJi6RqCmMiS3AWXt6Q][inet[/10.3.2.32:19300]]]; nested: RemoteTransportException[[Authority][inet[/10.3.2.31:19300]][index/shard/recovery/startRecovery]]; nested: RecoveryEngineException[[ezb][0] Phase[1] Execution failed]; nested: RecoverFilesRecoveryException[[ezb][0] Failed to transfer [1] files with total size of [76mb]]; nested: ReceiveTimeoutTransportException[[Collector][inet[/10.3.2.32:19300]][index/shard/recovery/cleanFiles] request_id [4901909] timed out after [900001ms]]; ]] | |
| [2013-12-17 15:26:10,280][INFO ][action.admin.cluster.node.shutdown] [Collector] shutting down in [200ms] | |
| [2013-12-17 15:26:10,488][INFO ][action.admin.cluster.node.shutdown] [Collector] initiating requested shutdown... | |
| [2013-12-17 15:26:10,500][INFO ][node ] [Collector] stopping ... | |
| [2013-12-17 15:26:10,821][WARN ][transport.netty ] [Collector] exception caught on transport layer [[id: 0x535546a2, /10.3.2.31:58621 => /10.3.2.32:19300]], closing connection | |
| java.io.IOException: Die Verbindung wurde vom Kommunikationspartner zurückgesetzt | |
| at sun.nio.ch.FileDispatcherImpl.read0(Native Method) | |
| at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) | |
| at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) | |
| at sun.nio.ch.IOUtil.read(IOUtil.java:192) | |
| at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:375) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:64) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| [2013-12-17 15:26:10,870][DEBUG][action.search.type ] [Collector] [3363698] Failed to execute query phase | |
| org.elasticsearch.search.SearchContextMissingException: No search context found for id [3363698] | |
| at org.elasticsearch.search.SearchService.findContext(SearchService.java:446) | |
| at org.elasticsearch.search.SearchService.executeScan(SearchService.java:197) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:474) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.executePhase(TransportSearchScrollScanAction.java:210) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction$2.run(TransportSearchScrollScanAction.java:180) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| [2013-12-17 15:26:10,871][DEBUG][action.search.type ] [Collector] [2324888] Failed to execute query phase | |
| org.elasticsearch.transport.SendRequestTransportException: [Authority][inet[/10.3.2.31:19300]][search/phase/scan/scroll] | |
| at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:202) | |
| at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:173) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:480) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.executePhase(TransportSearchScrollScanAction.java:210) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.start(TransportSearchScrollScanAction.java:139) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction.execute(TransportSearchScrollScanAction.java:71) | |
| at org.elasticsearch.action.search.TransportSearchScrollAction.doExecute(TransportSearchScrollAction.java:72) | |
| at org.elasticsearch.action.search.TransportSearchScrollAction.doExecute(TransportSearchScrollAction.java:42) | |
| at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:63) | |
| at org.elasticsearch.action.search.TransportSearchScrollAction$TransportHandler.messageReceived(TransportSearchScrollAction.java:92) | |
| at org.elasticsearch.action.search.TransportSearchScrollAction$TransportHandler.messageReceived(TransportSearchScrollAction.java:81) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handleRequest(MessageChannelHandler.java:212) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:109) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.transport.NodeNotConnectedException: [Authority][inet[/10.3.2.31:19300]] Node not connected | |
| at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:834) | |
| at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:532) | |
| at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:189) | |
| ... 37 more | |
| [2013-12-17 15:26:10,871][DEBUG][action.search.type ] [Collector] [2324867] Failed to execute query phase | |
| org.elasticsearch.transport.SendRequestTransportException: [Authority][inet[/10.3.2.31:19300]][search/phase/scan/scroll] | |
| at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:202) | |
| at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:173) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:480) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.executePhase(TransportSearchScrollScanAction.java:210) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.start(TransportSearchScrollScanAction.java:139) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction.execute(TransportSearchScrollScanAction.java:71) | |
| at org.elasticsearch.action.search.TransportSearchScrollAction.doExecute(TransportSearchScrollAction.java:72) | |
| at org.elasticsearch.action.search.TransportSearchScrollAction.doExecute(TransportSearchScrollAction.java:42) | |
| at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:63) | |
| at org.elasticsearch.action.search.TransportSearchScrollAction$TransportHandler.messageReceived(TransportSearchScrollAction.java:92) | |
| at org.elasticsearch.action.search.TransportSearchScrollAction$TransportHandler.messageReceived(TransportSearchScrollAction.java:81) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handleRequest(MessageChannelHandler.java:212) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:109) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.transport.NodeNotConnectedException: [Authority][inet[/10.3.2.31:19300]] Node not connected | |
| at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:834) | |
| at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:532) | |
| at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:189) | |
| ... 37 more | |
| [2013-12-17 15:26:10,871][DEBUG][action.search.type ] [Collector] [3363694] Failed to execute query phase | |
| org.elasticsearch.search.SearchContextMissingException: No search context found for id [3363694] | |
| at org.elasticsearch.search.SearchService.findContext(SearchService.java:446) | |
| at org.elasticsearch.search.SearchService.executeScan(SearchService.java:197) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:474) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.executePhase(TransportSearchScrollScanAction.java:210) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction$2.run(TransportSearchScrollScanAction.java:180) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| [2013-12-17 15:26:10,870][DEBUG][action.search.type ] [Collector] [3363697] Failed to execute query phase | |
| org.elasticsearch.search.SearchContextMissingException: No search context found for id [3363697] | |
| at org.elasticsearch.search.SearchService.findContext(SearchService.java:446) | |
| at org.elasticsearch.search.SearchService.executeScan(SearchService.java:197) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:474) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.executePhase(TransportSearchScrollScanAction.java:210) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction$2.run(TransportSearchScrollScanAction.java:180) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| [2013-12-17 15:26:10,871][DEBUG][action.search.type ] [Collector] [2324891] Failed to execute query phase | |
| org.elasticsearch.transport.SendRequestTransportException: [Authority][inet[/10.3.2.31:19300]][search/phase/scan/scroll] | |
| at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:202) | |
| at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:173) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:480) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.executePhase(TransportSearchScrollScanAction.java:210) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.start(TransportSearchScrollScanAction.java:139) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction.execute(TransportSearchScrollScanAction.java:71) | |
| at org.elasticsearch.action.search.TransportSearchScrollAction.doExecute(TransportSearchScrollAction.java:72) | |
| at org.elasticsearch.action.search.TransportSearchScrollAction.doExecute(TransportSearchScrollAction.java:42) | |
| at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:63) | |
| at org.elasticsearch.action.search.TransportSearchScrollAction$TransportHandler.messageReceived(TransportSearchScrollAction.java:92) | |
| at org.elasticsearch.action.search.TransportSearchScrollAction$TransportHandler.messageReceived(TransportSearchScrollAction.java:81) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handleRequest(MessageChannelHandler.java:212) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:109) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.transport.NodeNotConnectedException: [Authority][inet[/10.3.2.31:19300]] Node not connected | |
| at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:834) | |
| at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:532) | |
| at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:189) | |
| ... 37 more | |
| [2013-12-17 15:26:10,870][DEBUG][action.search.type ] [Collector] Node [ldtEHWFxRcaZBHQaKFS_TQ] not available for scroll request [c2Nhbjs0OzMzNjM2NzM6X3ZYVm1KaTZScUNtTWlTM0FXWHQ2UTsyMzI0ODY3OllOZEdjZ29RVHFpZ0xtczMxWUNLTmc7MzQ1OTU1NjpsZHRFSFdGeFJjYVpCSFFhS0ZTX1RROzMzNjM2NzI6X3ZYVm1KaTZScUNtTWlTM0FXWHQ2UTsxO3RvdGFsX2hpdHM6MTs=] | |
| [2013-12-17 15:26:10,871][DEBUG][action.search.type ] [Collector] [3363693] Failed to execute query phase | |
| org.elasticsearch.search.SearchContextMissingException: No search context found for id [3363693] | |
| at org.elasticsearch.search.SearchService.findContext(SearchService.java:446) | |
| at org.elasticsearch.search.SearchService.executeScan(SearchService.java:197) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:474) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.executePhase(TransportSearchScrollScanAction.java:210) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction$2.run(TransportSearchScrollScanAction.java:180) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| [2013-12-17 15:26:10,925][DEBUG][action.search.type ] [Collector] [3363672] Failed to execute query phase | |
| org.elasticsearch.search.SearchContextMissingException: No search context found for id [3363672] | |
| at org.elasticsearch.search.SearchService.findContext(SearchService.java:446) | |
| at org.elasticsearch.search.SearchService.executeScan(SearchService.java:197) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:474) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.executePhase(TransportSearchScrollScanAction.java:210) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction$2.run(TransportSearchScrollScanAction.java:180) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| [2013-12-17 15:26:10,925][DEBUG][action.search.type ] [Collector] [3363673] Failed to execute query phase | |
| org.elasticsearch.search.SearchContextMissingException: No search context found for id [3363673] | |
| at org.elasticsearch.search.SearchService.findContext(SearchService.java:446) | |
| at org.elasticsearch.search.SearchService.executeScan(SearchService.java:197) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:474) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.executePhase(TransportSearchScrollScanAction.java:210) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction$2.run(TransportSearchScrollScanAction.java:180) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| [2013-12-17 15:26:10,925][DEBUG][action.search.type ] [Collector] Node [ldtEHWFxRcaZBHQaKFS_TQ] not available for scroll request [c2Nhbjs0OzMzNjM2NzM6X3ZYVm1KaTZScUNtTWlTM0FXWHQ2UTsyMzI0ODY3OllOZEdjZ29RVHFpZ0xtczMxWUNLTmc7MzQ1OTU1NjpsZHRFSFdGeFJjYVpCSFFhS0ZTX1RROzMzNjM2NzI6X3ZYVm1KaTZScUNtTWlTM0FXWHQ2UTsxO3RvdGFsX2hpdHM6MTs=] | |
| [2013-12-17 15:26:10,894][DEBUG][action.search.type ] [Collector] All shards failed for phase: [init_scan] | |
| [2013-12-17 15:26:10,888][DEBUG][action.search.type ] [Collector] [zdb][2], node[YNdGcgoQTqigLms31YCKNg], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.search.SearchRequest@78a50cc1] | |
| org.elasticsearch.transport.SendRequestTransportException: [Authority][inet[/10.3.2.31:19300]][search/phase/scan] | |
| at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:202) | |
| at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:173) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:446) | |
| at org.elasticsearch.action.search.type.TransportSearchScanAction$AsyncAction.sendExecuteFirstPhase(TransportSearchScanAction.java:68) | |
| at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:216) | |
| at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.performFirstPhase(TransportSearchTypeAction.java:203) | |
| at org.elasticsearch.action.search.type.TransportSearchTypeAction$BaseAsyncAction.start(TransportSearchTypeAction.java:143) | |
| at org.elasticsearch.action.search.type.TransportSearchScanAction.doExecute(TransportSearchScanAction.java:52) | |
| at org.elasticsearch.action.search.type.TransportSearchScanAction.doExecute(TransportSearchScanAction.java:42) | |
| at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:63) | |
| at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:114) | |
| at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:43) | |
| at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:63) | |
| at org.elasticsearch.action.search.TransportSearchAction$TransportHandler.messageReceived(TransportSearchAction.java:135) | |
| at org.elasticsearch.action.search.TransportSearchAction$TransportHandler.messageReceived(TransportSearchAction.java:120) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handleRequest(MessageChannelHandler.java:212) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:109) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.transport.NodeNotConnectedException: [Authority][inet[/10.3.2.31:19300]] Node not connected | |
| at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:834) | |
| at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:532) | |
| at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:189) | |
| ... 41 more | |
| [2013-12-17 15:26:41,306][INFO ][node ] [Collector] stopped | |
| [2013-12-17 15:26:41,306][INFO ][node ] [Collector] closing ... | |
| [2013-12-17 15:26:51,692][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #105 | |
| [2013-12-17 15:26:51,692][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #106 | |
| [2013-12-17 15:26:51,692][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #107 | |
| [2013-12-17 15:26:51,692][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #108 | |
| [2013-12-17 15:26:51,692][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #109 | |
| [2013-12-17 15:26:51,692][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #110 | |
| [2013-12-17 15:26:51,693][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #111 | |
| [2013-12-17 15:26:51,693][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #112 | |
| [2013-12-17 15:26:51,693][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #113 | |
| [2013-12-17 15:26:51,693][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #114 | |
| [2013-12-17 15:26:51,693][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #115 | |
| [2013-12-17 15:26:51,693][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #116 | |
| [2013-12-17 15:26:51,693][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #117 | |
| [2013-12-17 15:26:51,693][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #118 | |
| [2013-12-17 15:26:51,693][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #119 | |
| [2013-12-17 15:26:51,693][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #120 | |
| [2013-12-17 15:26:51,693][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #121 | |
| [2013-12-17 15:26:51,694][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #122 | |
| [2013-12-17 15:26:51,694][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #123 | |
| [2013-12-17 15:26:51,694][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #124 | |
| [2013-12-17 15:26:51,694][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #125 | |
| [2013-12-17 15:26:51,694][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #126 | |
| [2013-12-17 15:26:51,694][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #127 | |
| [2013-12-17 15:26:51,694][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #128 | |
| [2013-12-17 15:26:51,694][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #129 | |
| [2013-12-17 15:26:51,694][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #130 | |
| [2013-12-17 15:26:51,694][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #66 | |
| [2013-12-17 15:26:51,694][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #67 | |
| [2013-12-17 15:26:51,694][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #68 | |
| [2013-12-17 15:26:51,695][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #69 | |
| [2013-12-17 15:26:51,695][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #70 | |
| [2013-12-17 15:26:51,695][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #71 | |
| [2013-12-17 15:26:51,695][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #72 | |
| [2013-12-17 15:26:51,695][INFO ][node ] [Collector] closed | |
| [2013-12-17 15:26:51,695][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #73 | |
| [2013-12-17 15:26:51,695][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #74 | |
| [2013-12-17 15:26:51,695][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #75 | |
| [2013-12-17 15:26:51,695][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #76 | |
| [2013-12-17 15:26:51,695][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #77 | |
| [2013-12-17 15:26:51,695][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #78 | |
| [2013-12-17 15:26:51,695][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #79 | |
| [2013-12-17 15:26:51,695][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #80 | |
| [2013-12-17 15:26:51,696][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #81 | |
| [2013-12-17 15:26:51,696][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #82 | |
| [2013-12-17 15:26:51,696][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #83 | |
| [2013-12-17 15:26:51,696][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #84 | |
| [2013-12-17 15:26:51,696][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #85 | |
| [2013-12-17 15:26:51,696][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #86 | |
| [2013-12-17 15:26:51,696][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #87 | |
| [2013-12-17 15:26:51,696][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #88 | |
| [2013-12-17 15:26:51,696][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #89 | |
| [2013-12-17 15:26:51,696][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #90 | |
| [2013-12-17 15:26:51,696][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #91 | |
| [2013-12-17 15:26:51,696][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #92 | |
| [2013-12-17 15:26:51,696][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #93 | |
| [2013-12-17 15:26:51,697][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #94 | |
| [2013-12-17 15:26:51,697][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #95 | |
| [2013-12-17 15:26:51,697][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #96 | |
| [2013-12-17 15:26:51,697][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #97 | |
| [2013-12-17 15:26:51,697][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #98 | |
| [2013-12-17 15:26:51,697][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #99 | |
| [2013-12-17 15:26:51,697][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #100 | |
| [2013-12-17 15:26:51,697][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #101 | |
| [2013-12-17 15:26:51,697][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #102 | |
| [2013-12-17 15:26:51,697][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #103 | |
| [2013-12-17 15:26:51,697][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #104 | |
| [2013-12-17 15:26:51,697][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #105 | |
| [2013-12-17 15:26:51,697][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #106 | |
| [2013-12-17 15:26:51,697][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #107 | |
| [2013-12-17 15:26:51,698][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #108 | |
| [2013-12-17 15:26:51,698][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #109 | |
| [2013-12-17 15:26:51,698][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #110 | |
| [2013-12-17 15:26:51,698][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #111 | |
| [2013-12-17 15:26:51,698][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #112 | |
| [2013-12-17 15:26:51,698][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #113 | |
| [2013-12-17 15:26:51,698][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #114 | |
| [2013-12-17 15:26:51,698][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #115 | |
| [2013-12-17 15:26:51,698][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #116 | |
| [2013-12-17 15:26:51,698][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #117 | |
| [2013-12-17 15:26:51,698][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #118 | |
| [2013-12-17 15:26:51,698][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #119 | |
| [2013-12-17 15:26:51,698][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #120 | |
| [2013-12-17 15:26:51,698][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #121 | |
| [2013-12-17 15:26:51,698][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #122 | |
| [2013-12-17 15:26:51,698][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #123 | |
| [2013-12-17 15:26:51,699][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #124 | |
| [2013-12-17 15:26:51,699][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #125 | |
| [2013-12-17 15:26:51,699][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #126 | |
| [2013-12-17 15:26:51,699][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #127 | |
| [2013-12-17 15:26:51,699][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #128 | |
| [2013-12-17 15:26:51,699][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #129 | |
| [2013-12-17 15:26:51,700][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #65 | |
| [2013-12-17 15:26:51,701][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #1 | |
| [2013-12-17 15:26:51,701][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #2 | |
| [2013-12-17 15:26:51,701][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #3 | |
| [2013-12-17 15:26:51,701][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #4 | |
| [2013-12-17 15:26:51,701][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #5 | |
| [2013-12-17 15:26:51,701][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #6 | |
| [2013-12-17 15:26:51,701][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #7 | |
| [2013-12-17 15:26:51,702][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #8 | |
| [2013-12-17 15:26:51,702][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #9 | |
| [2013-12-17 15:26:51,702][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #10 | |
| [2013-12-17 15:26:51,702][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #11 | |
| [2013-12-17 15:26:51,702][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #12 | |
| [2013-12-17 15:26:51,702][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #13 | |
| [2013-12-17 15:26:51,702][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #14 | |
| [2013-12-17 15:26:51,702][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #15 | |
| [2013-12-17 15:26:51,702][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #16 | |
| [2013-12-17 15:26:51,702][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #17 | |
| [2013-12-17 15:26:51,703][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #18 | |
| [2013-12-17 15:26:51,703][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #19 | |
| [2013-12-17 15:26:51,703][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #20 | |
| [2013-12-17 15:26:51,703][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #21 | |
| [2013-12-17 15:26:51,703][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #22 | |
| [2013-12-17 15:26:51,703][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #23 | |
| [2013-12-17 15:26:51,703][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #24 | |
| [2013-12-17 15:26:51,704][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #25 | |
| [2013-12-17 15:26:51,704][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #26 | |
| [2013-12-17 15:26:51,704][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #27 | |
| [2013-12-17 15:26:51,704][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #28 | |
| [2013-12-17 15:26:51,704][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #29 | |
| [2013-12-17 15:26:51,704][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #30 | |
| [2013-12-17 15:26:51,704][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #31 | |
| [2013-12-17 15:26:51,704][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #32 | |
| [2013-12-17 15:26:51,704][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #33 | |
| [2013-12-17 15:26:51,704][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #34 | |
| [2013-12-17 15:26:51,705][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #35 | |
| [2013-12-17 15:26:51,705][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #36 | |
| [2013-12-17 15:26:51,705][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #37 | |
| [2013-12-17 15:26:51,705][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #38 | |
| [2013-12-17 15:26:51,705][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #39 | |
| [2013-12-17 15:26:51,705][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #40 | |
| [2013-12-17 15:26:51,705][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #41 | |
| [2013-12-17 15:26:51,705][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #42 | |
| [2013-12-17 15:26:51,705][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #43 | |
| [2013-12-17 15:26:51,705][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #44 | |
| [2013-12-17 15:26:51,706][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #45 | |
| [2013-12-17 15:26:51,706][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #46 | |
| [2013-12-17 15:26:51,706][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #47 | |
| [2013-12-17 15:26:51,706][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #48 | |
| [2013-12-17 15:26:51,706][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #49 | |
| [2013-12-17 15:26:51,706][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #50 | |
| [2013-12-17 15:26:51,706][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #51 | |
| [2013-12-17 15:26:51,706][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #52 | |
| [2013-12-17 15:26:51,706][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #53 | |
| [2013-12-17 15:26:51,707][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #54 | |
| [2013-12-17 15:26:51,707][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #55 | |
| [2013-12-17 15:26:51,707][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #56 | |
| [2013-12-17 15:26:51,707][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #57 | |
| [2013-12-17 15:26:51,707][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #58 | |
| [2013-12-17 15:26:51,707][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #59 | |
| [2013-12-17 15:26:51,707][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #60 | |
| [2013-12-17 15:26:51,707][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #61 | |
| [2013-12-17 15:26:51,707][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #62 | |
| [2013-12-17 15:26:51,707][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #63 | |
| [2013-12-17 15:26:51,708][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #64 | |
| [2013-12-17 15:26:51,708][ERROR][netty.channel.socket.nio.AbstractNioSelector] Interrupted while wait for resources to be released #65 |
This file has been truncated, but you can view the full file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [2013-12-17 13:06:35,241][INFO ][node ] [The Profile] version[1.0.0.Beta2], pid[24704], build[296cfbe/2013-12-02T15:46:27Z] | |
| [2013-12-17 13:06:35,242][INFO ][node ] [The Profile] initializing ... | |
| [2013-12-17 13:06:35,260][INFO ][plugins ] [The Profile] loaded [analysis-german, ingest], sites [] | |
| [2013-12-17 13:06:38,110][INFO ][node ] [The Profile] initialized | |
| [2013-12-17 13:06:38,110][INFO ][node ] [The Profile] starting ... | |
| [2013-12-17 13:06:39,109][INFO ][transport ] [The Profile] bound_address {inet[/10.3.2.33:19300]}, publish_address {inet[/10.3.2.33:19300]} | |
| [2013-12-17 13:06:42,213][INFO ][cluster.service ] [The Profile] detected_master [Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]], added {[Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]],[Collector][_vXVmJi6RqCmMiS3AWXt6Q][inet[/10.3.2.32:19300]],}, reason: zen-disco-receive(from master [[Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]]]) | |
| [2013-12-17 13:06:42,250][INFO ][discovery ] [The Profile] zbn-1.0/ldtEHWFxRcaZBHQaKFS_TQ | |
| [2013-12-17 13:06:42,281][INFO ][http ] [The Profile] bound_address {inet[/10.3.2.33:19200]}, publish_address {inet[/10.3.2.33:19200]} | |
| [2013-12-17 13:06:42,282][INFO ][node ] [The Profile] started | |
| [2013-12-17 13:11:55,509][INFO ][index.shard.service ] [The Profile] [fix][0] updating refresh_interval from [1s] to [-1] | |
| [2013-12-17 13:11:55,510][INFO ][index.shard.service ] [The Profile] [fix][3] updating refresh_interval from [1s] to [-1] | |
| [2013-12-17 13:33:37,908][DEBUG][action.search.type ] [The Profile] [869553] Failed to execute query phase | |
| org.elasticsearch.search.SearchContextMissingException: No search context found for id [869553] | |
| at org.elasticsearch.search.SearchService.findContext(SearchService.java:446) | |
| at org.elasticsearch.search.SearchService.executeScan(SearchService.java:197) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:474) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.executePhase(TransportSearchScrollScanAction.java:210) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction$2.run(TransportSearchScrollScanAction.java:180) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| [2013-12-17 14:20:05,243][INFO ][monitor.jvm ] [The Profile] [gc][G1 Old Generation][4355][4] duration [5.1s], collections [1]/[5.3s], total [5.1s]/[11.2s], memory [3.9gb]->[2.9gb]/[4gb], all_pools {[Code Cache] [37.8mb]->[37.8mb]/[240mb]}{[Metaspace] [39.3mb]->[39.3mb]/[0b]}{[Compressed Class Space] [4.7mb]->[4.7mb]/[1gb]}{[G1 Eden Space] [14mb]->[4mb]/[0b]}{[G1 Survivor Space] [18mb]->[0b]/[0b]}{[G1 Old Gen] [3.8gb]->[2.9gb]/[4gb]} | |
| [2013-12-17 14:34:26,594][WARN ][transport ] [The Profile] Received response for a request that has timed out, sent [84831ms] ago, timed out [54831ms] ago, action [discovery/zen/fd/masterPing], node [[Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]]], id [4091718] | |
| [2013-12-17 14:34:26,607][WARN ][transport ] [The Profile] Received response for a request that has timed out, sent [54842ms] ago, timed out [24842ms] ago, action [discovery/zen/fd/masterPing], node [[Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]]], id [4091719] | |
| [2013-12-17 14:42:38,302][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:42:42,387][WARN ][transport.netty ] [The Profile] exception caught on transport layer [[id: 0x6fc78c8f, /10.3.2.31:33696 => /10.3.2.33:19300]], closing connection | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:42:38,313][DEBUG][action.search.type ] [The Profile] failed to reduce search | |
| org.elasticsearch.action.search.ReduceSearchPhaseException: Failed to execute phase [fetch], [reduce] | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.finishHim(TransportSearchScrollScanAction.java:241) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.access$1100(TransportSearchScrollScanAction.java:74) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction$3.onResult(TransportSearchScrollScanAction.java:215) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction$3.onResult(TransportSearchScrollScanAction.java:210) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction$12.handleResponse(SearchServiceTransportAction.java:489) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction$12.handleResponse(SearchServiceTransportAction.java:480) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.handleResponse(MessageChannelHandler.java:154) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:125) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) | |
| at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) | |
| at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) | |
| at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268) | |
| at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) | |
| at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) | |
| at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) | |
| at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) | |
| at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:42:38,313][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:42:38,313][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:42:38,313][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:42:38,313][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:42:38,314][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:42:38,312][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:42:38,312][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:42:38,312][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:42:38,312][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:42:38,312][WARN ][netty.channel.socket.nio.AbstractNioSelector] Unexpected exception in the selector loop. | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:42:38,313][DEBUG][action.search.type ] [The Profile] [3459569] Failed to execute query phase | |
| org.elasticsearch.ElasticSearchException: Java heap space | |
| at org.elasticsearch.ExceptionsHelper.convertToRuntime(ExceptionsHelper.java:37) | |
| at org.elasticsearch.search.SearchService.executeScan(SearchService.java:218) | |
| at org.elasticsearch.search.action.SearchServiceTransportAction.sendExecuteScan(SearchServiceTransportAction.java:474) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction.executePhase(TransportSearchScrollScanAction.java:210) | |
| at org.elasticsearch.action.search.type.TransportSearchScrollScanAction$AsyncAction$2.run(TransportSearchScrollScanAction.java:180) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:42:38,312][WARN ][transport.netty ] [The Profile] Failed to send error message back to client for action [search/phase/scan] | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:42:42,440][WARN ][transport.netty ] [The Profile] Actual Exception | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:45:40,456][INFO ][discovery.zen ] [The Profile] master_left [[Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]]], reason [failed to ping, tried [3] times, each with maximum [30s] timeout] | |
| [2013-12-17 14:45:40,468][INFO ][cluster.service ] [The Profile] master {new [Collector][_vXVmJi6RqCmMiS3AWXt6Q][inet[/10.3.2.32:19300]], previous [Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]]}, removed {[Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]],}, reason: zen-disco-master_failed ([Authority][YNdGcgoQTqigLms31YCKNg][inet[/10.3.2.31:19300]]) | |
| [2013-12-17 14:45:40,530][DEBUG][action.search.type ] [The Profile] [2324877] Failed to execute query phase | |
| org.elasticsearch.transport.NodeDisconnectedException: [Authority][inet[/10.3.2.31:19300]][search/phase/scan/scroll] disconnected | |
| [2013-12-17 14:45:40,530][DEBUG][action.search.type ] [The Profile] [2324881] Failed to execute query phase | |
| org.elasticsearch.transport.NodeDisconnectedException: [Authority][inet[/10.3.2.31:19300]][search/phase/scan/scroll] disconnected | |
| [2013-12-17 14:45:40,535][DEBUG][action.search.type ] [The Profile] [zdbholdings][1], node[YNdGcgoQTqigLms31YCKNg], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.search.SearchRequest@1ebfc6ab] | |
| org.elasticsearch.transport.NodeDisconnectedException: [Authority][inet[/10.3.2.31:19300]][search/phase/scan] disconnected | |
| [2013-12-17 14:45:40,577][WARN ][index.merge.scheduler ] [The Profile] [fix][3] failed to merge | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:45:40,584][WARN ][index.engine.robin ] [The Profile] [fix][3] failed engine | |
| org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: Java heap space | |
| at org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider$CustomConcurrentMergeScheduler.handleMergeException(ConcurrentMergeSchedulerProvider.java:109) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:518) | |
| Caused by: java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:45:40,592][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[STARTED], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [engine failure, message [MergeException[java.lang.OutOfMemoryError: Java heap space]; nested: OutOfMemoryError[Java heap space]; ]] | |
| [2013-12-17 14:45:41,510][INFO ][discovery.zen ] [The Profile] master_left [[Collector][_vXVmJi6RqCmMiS3AWXt6Q][inet[/10.3.2.32:19300]]], reason [no longer master] | |
| [2013-12-17 14:45:41,512][INFO ][cluster.service ] [The Profile] master {new [The Profile][ldtEHWFxRcaZBHQaKFS_TQ][inet[/10.3.2.33:19300]], previous [Collector][_vXVmJi6RqCmMiS3AWXt6Q][inet[/10.3.2.32:19300]]}, removed {[Collector][_vXVmJi6RqCmMiS3AWXt6Q][inet[/10.3.2.32:19300]],}, reason: zen-disco-master_failed ([Collector][_vXVmJi6RqCmMiS3AWXt6Q][inet[/10.3.2.32:19300]]) | |
| [2013-12-17 14:45:41,517][WARN ][indices.cluster ] [The Profile] [fix][3] master [[The Profile][ldtEHWFxRcaZBHQaKFS_TQ][inet[/10.3.2.33:19300]]] marked shard as started, but shard has not been created, mark shard as failed | |
| [2013-12-17 14:45:41,517][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[STARTED], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [master [The Profile][ldtEHWFxRcaZBHQaKFS_TQ][inet[/10.3.2.33:19300]] marked shard as started, but shard has not been created, mark shard as failed] | |
| [2013-12-17 14:45:41,521][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[STARTED], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [master [The Profile][ldtEHWFxRcaZBHQaKFS_TQ][inet[/10.3.2.33:19300]] marked shard as started, but shard has not been created, mark shard as failed] | |
| [2013-12-17 14:45:41,571][DEBUG][action.search.type ] [The Profile] [3363685] Failed to execute query phase | |
| org.elasticsearch.transport.NodeDisconnectedException: [Collector][inet[/10.3.2.32:19300]][search/phase/scan/scroll] disconnected | |
| [2013-12-17 14:45:41,571][DEBUG][action.search.type ] [The Profile] [zdbholdings][3], node[_vXVmJi6RqCmMiS3AWXt6Q], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.search.SearchRequest@1ebfc6ab] | |
| org.elasticsearch.transport.NodeDisconnectedException: [Collector][inet[/10.3.2.32:19300]][search/phase/scan] disconnected | |
| [2013-12-17 14:58:51,808][WARN ][index.merge.scheduler ] [The Profile] [fix][0] failed to merge | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:58:51,809][WARN ][index.engine.robin ] [The Profile] [fix][0] failed engine | |
| org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: Java heap space | |
| at org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider$CustomConcurrentMergeScheduler.handleMergeException(ConcurrentMergeSchedulerProvider.java:109) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:518) | |
| Caused by: java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 14:59:04,653][WARN ][index.translog ] [The Profile] [fix][0] failed to flush shard on translog threshold | |
| org.elasticsearch.index.engine.FlushFailedEngineException: [fix][0] Flush failed | |
| at org.elasticsearch.index.engine.robin.RobinEngine.flush(RobinEngine.java:853) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.flush(InternalIndexShard.java:591) | |
| at org.elasticsearch.index.translog.TranslogService$TranslogBasedFlush$1.run(TranslogService.java:186) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: java.lang.IllegalStateException: this writer hit an OutOfMemoryError; cannot commit | |
| at org.apache.lucene.index.IndexWriter.startCommit(IndexWriter.java:4340) | |
| at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:2877) | |
| at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:2970) | |
| at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:2940) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.flush(RobinEngine.java:839) | |
| ... 5 more | |
| [2013-12-17 14:59:04,653][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[STARTED], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [engine failure, message [MergeException[java.lang.OutOfMemoryError: Java heap space]; nested: OutOfMemoryError[Java heap space]; ]] | |
| [2013-12-17 14:59:04,655][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[STARTED], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [engine failure, message [MergeException[java.lang.OutOfMemoryError: Java heap space]; nested: OutOfMemoryError[Java heap space]; ]] | |
| [2013-12-17 14:59:04,959][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 14:59:04,959][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:59:04,960][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 14:59:04,961][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 14:59:10,314][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 14:59:10,314][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 14:59:10,315][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 14:59:10,315][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:12,931][WARN ][index.merge.scheduler ] [The Profile] [fix][0] failed to merge | |
| java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 15:17:12,941][WARN ][index.engine.robin ] [The Profile] [fix][0] failed engine | |
| org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: Java heap space | |
| at org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider$CustomConcurrentMergeScheduler.handleMergeException(ConcurrentMergeSchedulerProvider.java:109) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:518) | |
| Caused by: java.lang.OutOfMemoryError: Java heap space | |
| [2013-12-17 15:17:12,960][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[STARTED], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [engine failure, message [MergeException[java.lang.OutOfMemoryError: Java heap space]; nested: OutOfMemoryError[Java heap space]; ]] | |
| [2013-12-17 15:17:12,961][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[STARTED], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [engine failure, message [MergeException[java.lang.OutOfMemoryError: Java heap space]; nested: OutOfMemoryError[Java heap space]; ]] | |
| [2013-12-17 15:17:13,216][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:13,220][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:13,221][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:13,221][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:17,748][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:17,748][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:17,750][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:17,750][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:17,792][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:17,792][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:17,793][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:17,793][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:17,840][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:17,840][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:17,842][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:17,842][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:17,883][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:17,883][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:17,884][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:17,884][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:17,937][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:17,937][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:17,938][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:17,938][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:17,971][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:17,972][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:17,972][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:17,973][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:18,013][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:18,013][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:18,014][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:18,014][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:18,050][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:18,051][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:18,052][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:18,052][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:20,074][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:20,074][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:20,075][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:20,075][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:20,112][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:20,112][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:20,113][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:20,113][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:20,153][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:20,154][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:20,155][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:20,155][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:20,188][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:20,188][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:20,189][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:20,189][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:20,232][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:20,232][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:20,233][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:20,233][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:20,266][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:20,266][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:20,267][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:20,267][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:20,341][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:20,342][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:20,343][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:20,343][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:22,361][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:22,361][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:22,362][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:22,363][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:22,408][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:22,409][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:22,430][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:22,430][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:22,467][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:22,467][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:22,468][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:22,468][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:22,507][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:22,507][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:22,508][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:22,508][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:22,540][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:22,540][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:22,541][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:22,541][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:22,579][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:22,580][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:22,580][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:22,580][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:22,612][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:22,613][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:22,613][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:22,613][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:24,657][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:24,657][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:24,658][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:24,658][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:24,695][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:24,695][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:24,696][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:24,696][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:24,738][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:24,739][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:24,739][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:24,739][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:24,772][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:24,772][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:24,773][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:24,773][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:24,811][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:24,812][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:24,812][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:24,812][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:24,844][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:24,845][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:24,845][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:24,845][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:24,887][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:24,888][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:24,888][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:24,888][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:24,921][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:24,921][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:24,922][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:24,922][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:26,973][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:26,973][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:26,974][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:26,974][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:27,014][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:27,014][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:27,015][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:27,015][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:27,056][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:27,056][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:27,057][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:27,057][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:27,089][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:27,090][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:27,090][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:27,090][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:27,129][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:27,129][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:27,130][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:27,130][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:27,165][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:27,165][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:27,166][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:27,166][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:27,210][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:27,210][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:27,234][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:27,234][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:27,273][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:27,273][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:27,274][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:27,274][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:29,309][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:29,310][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:29,311][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:29,311][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:29,349][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:29,349][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:29,350][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:29,350][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:29,394][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:29,394][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:29,395][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:29,395][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:29,434][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:29,434][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:29,435][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:29,435][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:29,481][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:29,481][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:29,482][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:29,482][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:29,519][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:29,520][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:29,521][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:29,521][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:29,564][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:29,564][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:29,564][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:29,564][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:29,642][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:29,642][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:29,642][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:29,642][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:31,598][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:31,598][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:31,599][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:31,599][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:31,635][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:31,636][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:31,637][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:31,637][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:31,688][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:31,688][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:31,689][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:31,689][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:31,726][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:31,727][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:31,727][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:31,727][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:31,776][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:31,776][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:31,777][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:31,777][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:31,813][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:31,813][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:31,814][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:31,814][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:33,782][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:33,782][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:33,783][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:33,783][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:33,824][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:33,824][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:33,825][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:33,825][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:33,868][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:33,868][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:33,869][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:33,869][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:33,913][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:33,913][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:33,914][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:33,914][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:33,957][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:33,957][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:33,957][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:33,957][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:33,993][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:33,993][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:33,993][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:33,994][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:34,043][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:34,043][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:34,044][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:34,044][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:34,111][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:34,111][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:34,112][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:34,112][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:36,066][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:36,067][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:36,067][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:36,067][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:36,104][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:36,104][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:36,105][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:36,105][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:36,148][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:36,149][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:36,153][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:36,153][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:36,193][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:36,193][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:36,193][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:36,193][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:36,241][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:36,241][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:36,242][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:36,242][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:36,286][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:36,286][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:36,287][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:36,287][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:36,333][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:36,333][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:36,334][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:36,334][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:38,382][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:38,382][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:38,383][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:38,383][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:38,435][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:38,435][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:38,435][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:38,435][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:38,472][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:38,472][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:38,473][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:38,473][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:38,517][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:38,518][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:38,518][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:38,518][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:38,560][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:38,561][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:38,561][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:38,561][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:38,611][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:38,611][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:38,612][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:38,612][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:38,653][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:38,654][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:38,654][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:38,654][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:40,686][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:40,687][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:40,688][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:40,688][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:40,738][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:40,738][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:40,738][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:40,739][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:40,779][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:40,780][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:40,780][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:40,780][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:40,814][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:40,814][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:40,815][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:40,815][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:40,858][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:40,858][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:40,859][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:40,859][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:40,894][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:40,894][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:40,895][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:40,895][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:42,935][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:42,935][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:42,936][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:42,936][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:42,975][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:42,975][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:42,976][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:42,976][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:43,020][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:43,020][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:43,021][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:43,021][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:43,055][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:43,055][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:43,056][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:43,056][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:43,099][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:43,099][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:43,100][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:43,100][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:43,133][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:43,133][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:43,134][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:43,134][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:45,182][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:45,182][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:45,183][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:45,183][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:45,222][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:45,223][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:45,223][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:45,224][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:45,272][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:45,272][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:45,273][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:45,273][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:45,312][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:45,312][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:45,313][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:45,313][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:45,360][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:45,360][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:45,361][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:45,361][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:45,398][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:45,398][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:45,399][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:45,399][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:45,445][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:45,446][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:45,446][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:45,446][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:47,498][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:47,498][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:47,499][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:47,499][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:47,545][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:47,545][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:47,545][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:47,545][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:47,582][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:47,582][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:47,583][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:47,583][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:47,625][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:47,625][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:47,626][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:47,626][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:47,663][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:47,663][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:47,664][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:47,664][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:47,707][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:47,707][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:47,708][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:47,708][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:49,763][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:49,763][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:49,764][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:49,764][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:49,806][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:49,806][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:49,807][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:49,807][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:49,842][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:49,842][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:49,843][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:49,843][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:49,886][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:49,886][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:49,887][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:49,887][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:49,922][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:49,922][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:49,923][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:49,923][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:49,966][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:49,967][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:49,967][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:49,967][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:51,994][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:51,994][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:51,995][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:51,995][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:52,036][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:52,036][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:52,036][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:52,037][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:52,072][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:52,072][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:52,072][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:52,072][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:52,117][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:52,118][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:52,118][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:52,118][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:52,153][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:52,154][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:52,154][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:52,154][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:52,196][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:52,197][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:52,197][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:52,197][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:54,244][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:54,244][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:54,245][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:54,245][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:54,290][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:54,290][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:54,291][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:54,291][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:54,333][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:54,333][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:54,334][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:54,334][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:54,381][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:54,381][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:54,381][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:54,381][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:54,419][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:54,419][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:54,420][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:54,420][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:54,463][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:54,463][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:54,464][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:54,464][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:56,513][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:56,514][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:56,515][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:56,515][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:56,559][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:56,559][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:56,559][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:56,559][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:56,595][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:56,595][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:56,597][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:56,597][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:56,640][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:56,640][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:56,641][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:56,641][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:56,676][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:56,676][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:56,677][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:56,677][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:56,719][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:56,719][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:56,720][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:56,720][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:58,755][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:58,756][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:58,756][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:58,756][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:58,798][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:58,799][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:58,799][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:58,799][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:58,831][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:58,831][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:58,832][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:58,832][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:58,877][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:58,877][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:58,877][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:58,878][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:58,909][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:17:58,909][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:58,910][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:58,910][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:17:58,948][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:17:58,948][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:17:58,949][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:17:58,949][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:00,991][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:00,991][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:00,992][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:00,992][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:01,038][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:01,038][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:01,038][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:01,038][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:01,070][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:01,070][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:01,070][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:01,070][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:01,110][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:01,110][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:01,111][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:01,111][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:01,145][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:01,145][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:01,146][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:01,146][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:01,187][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:01,187][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:01,187][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:01,187][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:03,226][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:03,226][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:03,227][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:03,227][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:03,267][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:03,267][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:03,268][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:03,268][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:03,302][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:03,302][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:03,303][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:03,303][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:03,344][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:03,344][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:03,345][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:03,345][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:03,379][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:03,379][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:03,379][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:03,380][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:03,421][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:03,421][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:03,421][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:03,421][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:05,455][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:05,455][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:05,456][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:05,456][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:05,494][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:05,494][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:05,494][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:05,494][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:05,530][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:05,530][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:05,531][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:05,531][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:05,572][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:05,572][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:05,572][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:05,573][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:05,613][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:05,613][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:05,613][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:05,614][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:07,667][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:07,667][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:07,668][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:07,668][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:07,705][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:07,705][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:07,706][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:07,706][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:07,750][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:07,751][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:07,751][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:07,751][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:07,789][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:07,789][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:07,790][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:07,790][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:07,834][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:07,834][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:07,835][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:07,835][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:07,874][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:07,874][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:07,874][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:07,874][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:09,913][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:09,914][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:09,914][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:09,914][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:09,946][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:09,946][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:09,947][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:09,947][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:09,991][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:09,991][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:09,992][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:09,992][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:10,027][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:10,027][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:10,027][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:10,027][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:10,069][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:10,069][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:10,069][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:10,070][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:10,145][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:10,145][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:10,146][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:10,146][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:10,198][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:10,199][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:10,199][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:10,199][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:12,223][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:12,223][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:12,224][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:12,224][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:12,267][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:12,268][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:12,268][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:12,268][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:12,303][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:12,303][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:12,304][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:12,304][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:12,346][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:12,346][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:12,346][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:12,346][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:12,385][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:12,385][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:12,386][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:12,386][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:14,155][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:14,155][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:14,156][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:14,156][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:14,196][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:14,196][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:14,197][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:14,197][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:14,240][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:14,240][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:14,241][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:14,241][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:14,273][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:14,273][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:14,273][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:14,273][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:14,314][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:14,314][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:14,315][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:14,315][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:14,348][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:14,348][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:14,349][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:14,349][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:16,116][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:16,116][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:16,117][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:16,117][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:16,148][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:16,148][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:16,148][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:16,148][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:16,186][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:16,186][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:16,186][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:16,186][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:16,221][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:16,221][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:16,221][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:16,221][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:16,262][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:16,262][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:16,263][WARN ][cluster.action.shard ] [The Profile] [fix][0] sending failed shard for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:16,263][WARN ][cluster.action.shard ] [The Profile] [fix][0] received shard failed for [fix][0], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][0] failed recovery]; nested: EngineCreationFailureException[[fix][0] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/0/index/write.lock]; ]] | |
| [2013-12-17 15:18:16,300][WARN ][index.engine.robin ] [The Profile] [fix][3] shard is locked, releasing lock | |
| [2013-12-17 15:18:16,300][WARN ][indices.cluster ] [The Profile] [fix][3] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][3] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][3] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:201) | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:189) | |
| ... 3 more | |
| Caused by: org.apache.lucene.store.LockReleaseFailedException: Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock | |
| at org.apache.lucene.store.NativeFSLock.release(NativeFSLockFactory.java:295) | |
| at org.apache.lucene.index.IndexWriter.unlock(IndexWriter.java:4458) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1338) | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:259) | |
| ... 6 more | |
| [2013-12-17 15:18:16,301][WARN ][cluster.action.shard ] [The Profile] [fix][3] sending failed shard for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:16,301][WARN ][cluster.action.shard ] [The Profile] [fix][3] received shard failed for [fix][3], node[ldtEHWFxRcaZBHQaKFS_TQ], [P], s[INITIALIZING], indexUUID [Zzwl8alYSp2rXIivaVcZjg], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[fix][3] failed recovery]; nested: EngineCreationFailureException[[fix][3] failed to create engine]; nested: LockReleaseFailedException[Cannot forcefully unlock a NativeFSLock which is held by another indexer component: /home/es/elasticsearch-1.0.0.Beta2/data/zbn-1.0/nodes/0/indices/fix/3/index/write.lock]; ]] | |
| [2013-12-17 15:18:18,080][WARN ][index.engine.robin ] [The Profile] [fix][0] shard is locked, releasing lock | |
| [2013-12-17 15:18:18,080][WARN ][indices.cluster ] [The Profile] [fix][0] failed to start shard | |
| org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [fix][0] failed recovery | |
| at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:248) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:744) | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [fix][0] failed to create engine | |
| at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:261) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:704) | |
| at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(Lo |
View raw
(Sorry about that, but we can’t show files that are this big right now.)
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment