elasticsearch.log

[root@ct-0088 ~]# ./bin/elasticsearch
[root@ct-0088 ~]# ./logs/elasticsearchStaging_4hr.log
[2013-07-12 14:07:56,405][INFO ][node                     ] [CT-0088] {0.90.0.RC2}[3049]: starting ...
[2013-07-12 14:07:56,567][INFO ][transport                ] [CT-0088] bound_address {inet[/0:0:0:0:0:0:0:0:9300]}, publish_address {inet[/192.168.50.171:9300]}
[2013-07-12 14:07:59,598][INFO ][cluster.service          ] [CT-0088] new_master [CT-0088][Zukt8ivLRd6LIzeZoriR8A][inet[/192.168.50.171:9300]]{max_local_storage_nodes=1}, reason: zen-disco-join (elected_as_master)
[2013-07-12 14:07:59,620][INFO ][discovery                ] [CT-0088] elasticsearchStaging_4hr/Zukt8ivLRd6LIzeZoriR8A
[2013-07-12 14:07:59,659][INFO ][http                     ] [CT-0088] bound_address {inet[/0:0:0:0:0:0:0:0:9200]}, publish_address {inet[/192.168.50.171:9200]}
[2013-07-12 14:07:59,660][INFO ][node                     ] [CT-0088] {0.90.0.RC2}[3049]: started
[2013-07-12 14:08:01,055][INFO ][gateway.local.state.meta ] [CT-0088] [mlivemass_back] dangling index, exists on local file system, but not in cluster metadata, scheduling to delete in [2h], auto import to cluster state [YES]
[2013-07-12 14:08:01,060][INFO ][gateway.local.state.meta ] [CT-0088] dangled index directory name is [mlivemass_back], state name is [mlivemass_ipdr_2], renaming to directory name
[2013-07-12 14:08:01,068][INFO ][gateway                  ] [CT-0088] recovered [8] indices into cluster_state
[2013-07-12 14:08:01,072][INFO ][gateway.local.state.meta ] [CT-0088] auto importing dangled indices [mlivemass_back/OPEN] from [[CT-0088][Zukt8ivLRd6LIzeZoriR8A][inet[/192.168.50.171:9300]]{max_local_storage_nodes=1}]

[2013-07-12 14:08:34,052][WARN ][indices.memory           ] [CT-0088] failed to set shard [mlivemass_ipdr_2][1] index buffer to [153.4mb]
[2013-07-12 14:08:34,052][WARN ][indices.cluster          ] [CT-0088] [mlivemass_ipdr_2][1] failed to start shard
org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [mlivemass_ipdr_2][1] failed recovery
	at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:225)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:895)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:918)
	at java.lang.Thread.run(Thread.java:662)
Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [mlivemass_ipdr_2][1] failed to open reader on writer
	at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:287)
	at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:570)
	at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:200)
	at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:174)
	... 3 more
Caused by: org.apache.lucene.index.CorruptIndexException: codec header mismatch: actual header=1095848538 vs expected header=1071082519 (resource: NIOFSIndexInput(path="/iSCSI_2/es_4r/elasticsearchStaging_4hr/nodes/0/indices/mlivemass_ipdr_2/1/index/_b6k_es090_0.tip"))
	at org.apache.lucene.codecs.CodecUtil.checkHeader(CodecUtil.java:128)
	at org.apache.lucene.codecs.BlockTreeTermsReader.readIndexHeader(BlockTreeTermsReader.java:206)
	at org.apache.lucene.codecs.BlockTreeTermsReader.<init>(BlockTreeTermsReader.java:132)
	at org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat.fieldsProducer(Lucene41PostingsFormat.java:437)
	at org.elasticsearch.index.codec.postingsformat.BloomFilterPostingsFormat$BloomFilteredFieldsProducer.<init>(BloomFilterPostingsFormat.java:129)
	at org.elasticsearch.index.codec.postingsformat.BloomFilterPostingsFormat.fieldsProducer(BloomFilterPostingsFormat.java:100)
	at org.elasticsearch.index.codec.postingsformat.ElasticSearch090PostingsFormat.fieldsProducer(ElasticSearch090PostingsFormat.java:81)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:194)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:233)
	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:127)
	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:56)
	at org.apache.lucene.index.ReadersAndLiveDocs.getReader(ReadersAndLiveDocs.java:121)
	at org.apache.lucene.index.ReadersAndLiveDocs.getReadOnlyClone(ReadersAndLiveDocs.java:218)
	at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:96)
	at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:369)
	at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:111)
	at org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:89)
	at org.elasticsearch.index.engine.robin.RobinEngine.buildSearchManager(RobinEngine.java:1412)
	at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:271)
	... 6 more
[2013-07-12 14:08:34,092][WARN ][indices.memory           ] [CT-0088] failed to set shard [mlivemass_ipdr_2][1] index buffer to [136.4mb]
[2013-07-12 14:08:34,094][WARN ][cluster.action.shard     ] [CT-0088] sending failed shard for [mlivemass_ipdr_2][1], node[Zukt8ivLRd6LIzeZoriR8A], [P], s[INITIALIZING], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[mlivemass_ipdr_2][1] failed recovery]; nested: EngineCreationFailureException[[mlivemass_ipdr_2][1] failed to open reader on writer]; nested: CorruptIndexException[codec header mismatch: actual header=1095848538 vs expected header=1071082519 (resource: NIOFSIndexInput(path="/iSCSI_2/es_4r/elasticsearchStaging_4hr/nodes/0/indices/mlivemass_ipdr_2/1/index/_b6k_es090_0.tip"))]; ]]
[2013-07-12 14:08:34,094][WARN ][cluster.action.shard     ] [CT-0088] received shard failed for [mlivemass_ipdr_2][1], node[Zukt8ivLRd6LIzeZoriR8A], [P], s[INITIALIZING], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[mlivemass_ipdr_2][1] failed recovery]; nested: EngineCreationFailureException[[mlivemass_ipdr_2][1] failed to open reader on writer]; nested: CorruptIndexException[codec header mismatch: actual header=1095848538 vs expected header=1071082519 (resource: NIOFSIndexInput(path="/iSCSI_2/es_4r/elasticsearchStaging_4hr/nodes/0/indices/mlivemass_ipdr_2/1/index/_b6k_es090_0.tip"))]; ]]
[2013-07-12 14:08:34,161][WARN ][indices.cluster          ] [CT-0088] [mlivemass_ipdr_2][1] failed to start shard
org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [mlivemass_ipdr_2][1] failed recovery
	at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:225)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:895)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:918)
	at java.lang.Thread.run(Thread.java:662)
Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [mlivemass_ipdr_2][1] failed to open reader on writer
	at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:287)
	at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:570)
	at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:200)
	at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:174)
	... 3 more
Caused by: org.apache.lucene.index.CorruptIndexException: codec header mismatch: actual header=1095848538 vs expected header=1071082519 (resource: NIOFSIndexInput(path="/iSCSI_2/es_4r/elasticsearchStaging_4hr/nodes/0/indices/mlivemass_ipdr_2/1/index/_b6k_es090_0.tip"))
	at org.apache.lucene.codecs.CodecUtil.checkHeader(CodecUtil.java:128)
	at org.apache.lucene.codecs.BlockTreeTermsReader.readIndexHeader(BlockTreeTermsReader.java:206)
	at org.apache.lucene.codecs.BlockTreeTermsReader.<init>(BlockTreeTermsReader.java:132)
	at org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat.fieldsProducer(Lucene41PostingsFormat.java:437)
	at org.elasticsearch.index.codec.postingsformat.BloomFilterPostingsFormat$BloomFilteredFieldsProducer.<init>(BloomFilterPostingsFormat.java:129)
	at org.elasticsearch.index.codec.postingsformat.BloomFilterPostingsFormat.fieldsProducer(BloomFilterPostingsFormat.java:100)
	at org.elasticsearch.index.codec.postingsformat.ElasticSearch090PostingsFormat.fieldsProducer(ElasticSearch090PostingsFormat.java:81)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:194)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:233)
	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:127)
	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:56)
	at org.apache.lucene.index.ReadersAndLiveDocs.getReader(ReadersAndLiveDocs.java:121)
	at org.apache.lucene.index.ReadersAndLiveDocs.getReadOnlyClone(ReadersAndLiveDocs.java:218)
	at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:96)
	at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:369)
	at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:111)
	at org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:89)
	at org.elasticsearch.index.engine.robin.RobinEngine.buildSearchManager(RobinEngine.java:1412)
	at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:271)
	... 6 more
[2013-07-12 14:08:34,161][WARN ][cluster.action.shard     ] [CT-0088] sending failed shard for [mlivemass_ipdr_2][1], node[Zukt8ivLRd6LIzeZoriR8A], [P], s[INITIALIZING], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[mlivemass_ipdr_2][1] failed recovery]; nested: EngineCreationFailureException[[mlivemass_ipdr_2][1] failed to open reader on writer]; nested: CorruptIndexException[codec header mismatch: actual header=1095848538 vs expected header=1071082519 (resource: NIOFSIndexInput(path="/iSCSI_2/es_4r/elasticsearchStaging_4hr/nodes/0/indices/mlivemass_ipdr_2/1/index/_b6k_es090_0.tip"))]; ]]
[2013-07-12 14:08:34,162][WARN ][cluster.action.shard     ] [CT-0088] received shard failed for [mlivemass_ipdr_2][1], node[Zukt8ivLRd6LIzeZoriR8A], [P], s[INITIALIZING], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[mlivemass_ipdr_2][1] failed recovery]; nested: EngineCreationFailureException[[mlivemass_ipdr_2][1] failed to open reader on writer]; nested: CorruptIndexException[codec header mismatch: actual header=1095848538 vs expected header=1071082519 (resource: NIOFSIndexInput(path="/iSCSI_2/es_4r/elasticsearchStaging_4hr/nodes/0/indices/mlivemass_ipdr_2/1/index/_b6k_es090_0.tip"))]; ]]
[2013-07-12 14:08:53,609][WARN ][indices.cluster          ] [CT-0088] [mlivemass_back][0] failed to create shard
org.elasticsearch.index.shard.IndexShardCreationException: [mlivemass_back][0] failed to create shard
	at org.elasticsearch.index.service.InternalIndexService.createShard(InternalIndexService.java:340)
	at org.elasticsearch.indices.cluster.IndicesClusterStateService.applyInitializingShard(IndicesClusterStateService.java:567)
	at org.elasticsearch.indices.cluster.IndicesClusterStateService.applyNewOrUpdatedShards(IndicesClusterStateService.java:532)
	at org.elasticsearch.indices.cluster.IndicesClusterStateService.clusterChanged(IndicesClusterStateService.java:174)
	at org.elasticsearch.cluster.service.InternalClusterService$2.run(InternalClusterService.java:321)
	at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:95)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:895)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:918)
	at java.lang.Thread.run(Thread.java:662)
Caused by: java.io.FileNotFoundException: _b3u.si
	at org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:265)
	at org.apache.lucene.store.RateLimitedFSDirectory.fileLength(RateLimitedFSDirectory.java:65)
	at org.elasticsearch.index.store.Store$StoreDirectory.<init>(Store.java:311)
	at org.elasticsearch.index.store.Store.<init>(Store.java:82)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
	at org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:54)
	at org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
	at org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:98)
	at org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:45)
	at org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:819)
	at org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:42)
	at org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:56)
	at org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:45)
	at org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
	at org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
	at org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
	at org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:98)
	at org.elasticsearch.common.inject.FactoryProxy.get(FactoryProxy.java:52)
	at org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:45)
	at org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:819)
	at org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:42)
	at org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:56)
	at org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:45)
	at org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:200)
	at org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:193)
	at org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:812)
	at org.elasticsearch.common.inject.InjectorBuilder.loadEagerSingletons(InjectorBuilder.java:193)
	at org.elasticsearch.common.inject.InjectorBuilder.injectDynamically(InjectorBuilder.java:175)
	at org.elasticsearch.common.inject.InjectorBuilder.build(InjectorBuilder.java:110)
	at org.elasticsearch.common.inject.InjectorImpl.createChildInjector(InjectorImpl.java:129)
	at org.elasticsearch.common.inject.ModulesBuilder.createChildInjector(ModulesBuilder.java:66)
	at org.elasticsearch.index.service.InternalIndexService.createShard(InternalIndexService.java:338)
	... 8 more
[2013-07-12 14:08:53,610][WARN ][cluster.action.shard     ] [CT-0088] sending failed shard for [mlivemass_back][0], node[Zukt8ivLRd6LIzeZoriR8A], [P], s[INITIALIZING], reason [Failed to create shard, message [IndexShardCreationException[[mlivemass_back][0] failed to create shard]; nested: FileNotFoundException[_b3u.si]; ]]
[2013-07-12 14:08:53,610][WARN ][cluster.action.shard     ] [CT-0088] received shard failed for [mlivemass_back][0], node[Zukt8ivLRd6LIzeZoriR8A], [P], s[INITIALIZING], reason [Failed to create shard, message [IndexShardCreationException[[mlivemass_back][0] failed to create shard]; nested: FileNotFoundException[_b3u.si]; ]]
[2013-07-12 14:09:00,756][WARN ][indices.cluster          ] [CT-0088] [mlivemass_back][0] failed to create shard