Elastic Search File Not Found Errors

gistfile1.txt

        ... 10 more
[2012-11-30 16:48:58,470][WARN ][index.gateway            ] [i-b41873ed] [lodgings][4] failed to snapshot (scheduled)
org.elasticsearch.index.gateway.IndexShardGatewaySnapshotFailedException: [lodgings][4] Failed to perform snapshot (index files)
        at org.elasticsearch.index.gateway.blobstore.BlobStoreIndexShardGateway.doSnapshot(BlobStoreIndexShardGateway.java:246)
        at org.elasticsearch.index.gateway.blobstore.BlobStoreIndexShardGateway.snapshot(BlobStoreIndexShardGateway.java:160)
        at org.elasticsearch.index.gateway.IndexShardGatewayService$2.snapshot(IndexShardGatewayService.java:271)
        at org.elasticsearch.index.gateway.IndexShardGatewayService$2.snapshot(IndexShardGatewayService.java:265)
        at org.elasticsearch.index.engine.robin.RobinEngine.snapshot(RobinEngine.java:1056)
        at org.elasticsearch.index.shard.service.InternalIndexShard.snapshot(InternalIndexShard.java:481)
        at org.elasticsearch.index.gateway.IndexShardGatewayService.snapshot(IndexShardGatewayService.java:265)
        at org.elasticsearch.index.gateway.IndexShardGatewayService$SnapshotRunnable.run(IndexShardGatewayService.java:366)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
        at java.lang.Thread.run(Thread.java:679)
Caused by: java.io.FileNotFoundException: /opt/data/elasticsearch/elasticsearch_cluster_3/nodes/0/indices/lodgings/4/index/_4i.fnm (No such file or directory)
        at java.io.RandomAccessFile.open(Native Method)
        at java.io.RandomAccessFile.<init>(RandomAccessFile.java:233)
        at org.apache.lucene.store.SimpleFSDirectory$SimpleFSIndexInput$Descriptor.<init>(SimpleFSDirectory.java:71)
        at org.apache.lucene.store.SimpleFSDirectory$SimpleFSIndexInput.<init>(SimpleFSDirectory.java:98)
        at org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.<init>(NIOFSDirectory.java:92)
        at org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:79)
        at org.apache.lucene.store.FSDirectory.openInput(FSDirectory.java:345)
        at org.elasticsearch.index.store.Store.openInputRaw(Store.java:314)
        at org.elasticsearch.index.gateway.blobstore.BlobStoreIndexShardGateway.snapshotFile(BlobStoreIndexShardGateway.java:755)
        at org.elasticsearch.index.gateway.blobstore.BlobStoreIndexShardGateway.doSnapshot(BlobStoreIndexShardGateway.java:228)
        ... 10 more
[2012-11-30 16:49:00,250][WARN ][index.gateway            ] [i-b41873ed] [lodgings][0] failed to snapshot (scheduled)
org.elasticsearch.index.gateway.IndexShardGatewaySnapshotFailedException: [lodgings][0] Failed to perform snapshot (index files)
        at org.elasticsearch.index.gateway.blobstore.BlobStoreIndexShardGateway.doSnapshot(BlobStoreIndexShardGateway.java:246)
        at org.elasticsearch.index.gateway.blobstore.BlobStoreIndexShardGateway.snapshot(BlobStoreIndexShardGateway.java:160)
        at org.elasticsearch.index.gateway.IndexShardGatewayService$2.snapshot(IndexShardGatewayService.java:271)
        at org.elasticsearch.index.gateway.IndexShardGatewayService$2.snapshot(IndexShardGatewayService.java:265)
        at org.elasticsearch.index.engine.robin.RobinEngine.snapshot(RobinEngine.java:1056)
        at org.elasticsearch.index.shard.service.InternalIndexShard.snapshot(InternalIndexShard.java:481)
        at org.elasticsearch.index.gateway.IndexShardGatewayService.snapshot(IndexShardGatewayService.java:265)
        at org.elasticsearch.index.gateway.IndexShardGatewayService$SnapshotRunnable.run(IndexShardGatewayService.java:366)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
        at java.lang.Thread.run(Thread.java:679)
Caused by: java.io.FileNotFoundException: /opt/data/elasticsearch/elasticsearch_cluster_3/nodes/0/indices/lodgings/0/index/_5w.fnm (No such file or directory)
        at java.io.RandomAccessFile.open(Native Method)
        at java.io.RandomAccessFile.<init>(RandomAccessFile.java:233)
        at org.apache.lucene.store.SimpleFSDirectory$SimpleFSIndexInput$Descriptor.<init>(SimpleFSDirectory.java:71)
        at org.apache.lucene.store.SimpleFSDirectory$SimpleFSIndexInput.<init>(SimpleFSDirectory.java:98)
        at org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.<init>(NIOFSDirectory.java:92)
        at org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:79)
        at org.apache.lucene.store.FSDirectory.openInput(FSDirectory.java:345)
        at org.elasticsearch.index.store.Store.openInputRaw(Store.java:314)
        at org.elasticsearch.index.gateway.blobstore.BlobStoreIndexShardGateway.snapshotFile(BlobStoreIndexShardGateway.java:755)
        at org.elasticsearch.index.gateway.blobstore.BlobStoreIndexShardGateway.doSnapshot(BlobStoreIndexShardGateway.java:228)
        ... 10 more
[2012-11-30 16:49:00,501][WARN ][index.gateway            ] [i-b41873ed] [lodgings][3] failed to snapshot (scheduled)
org.elasticsearch.index.gateway.IndexShardGatewaySnapshotFailedException: [lodgings][3] Failed to perform snapshot (index files)
        at org.elasticsearch.index.gateway.blobstore.BlobStoreIndexShardGateway.doSnapshot(BlobStoreIndexShardGateway.java:246)
        at org.elasticsearch.index.gateway.blobstore.BlobStoreIndexShardGateway.snapshot(BlobStoreIndexShardGateway.java:160)
        at org.elasticsearch.index.gateway.IndexShardGatewayService$2.snapshot(IndexShardGatewayService.java:271)
        at org.elasticsearch.index.gateway.IndexShardGatewayService$2.snapshot(IndexShardGatewayService.java:265)
        at org.elasticsearch.index.engine.robin.RobinEngine.snapshot(RobinEngine.java:1056)
        at org.elasticsearch.index.shard.service.InternalIndexShard.snapshot(InternalIndexShard.java:481)
        at org.elasticsearch.index.gateway.IndexShardGatewayService.snapshot(IndexShardGatewayService.java:265)
        at org.elasticsearch.index.gateway.IndexShardGatewayService$SnapshotRunnable.run(IndexShardGatewayService.java:366)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
        at java.lang.Thread.run(Thread.java:679)
Caused by: java.io.FileNotFoundException: /opt/data/elasticsearch/elasticsearch_cluster_3/nodes/0/indices/lodgings/3/index/segments_1 (No such file or directory)
        at java.io.RandomAccessFile.open(Native Method)
        at java.io.RandomAccessFile.<init>(RandomAccessFile.java:233)
        at org.apache.lucene.store.SimpleFSDirectory$SimpleFSIndexInput$Descriptor.<init>(SimpleFSDirectory.java:71)
        at org.apache.lucene.store.SimpleFSDirectory$SimpleFSIndexInput.<init>(SimpleFSDirectory.java:98)
        at org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.<init>(NIOFSDirectory.java:92)
        at org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:79)
        at org.apache.lucene.store.FSDirectory.openInput(FSDirectory.java:345)
        at org.elasticsearch.index.store.Store.openInputRaw(Store.java:314)
        at org.elasticsearch.index.gateway.blobstore.BlobStoreIndexShardGateway.snapshotFile(BlobStoreIndexShardGateway.java:755)
        at org.elasticsearch.index.gateway.blobstore.BlobStoreIndexShardGateway.doSnapshot(BlobStoreIndexShardGateway.java:228)
        ... 10 more
[2012-11-30 16:49:02,541][WARN ][index.engine.robin       ] [i-b41873ed] [places][1] failed to read latest segment infos on flush
java.io.FileNotFoundException: /opt/data/elasticsearch/elasticsearch_cluster_3/nodes/0/indices/places/1/index/segments_1 (No such file or directory)
:

indexing into cluster 3, a completely new cluster

elastic/elasticsearch-cloud-aws#15

Seems to not be unique to us.

http://elasticsearch-users.115913.n3.nabble.com/snapshot-failing-td4025681.html

Interesting.