ES problem while running test suite

elasticsearch-server.log

[2011-10-11 18:21:54,840][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] creating index, cause [api], shards [1]/[1], mappings []
[2011-10-11 18:21:54,854][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [document]
[2011-10-11 18:21:54,859][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [person]
[2011-10-11 18:21:54,862][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [source]
[2011-10-11 18:21:54,866][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [tag]
[2011-10-11 18:21:54,879][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] deleting index
[2011-10-11 18:21:54,883][WARN ][indices.cluster          ] [Zodiak] [cp_text_test][0] failed to start shard
org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [cp_text_test][0] failed recovery
	at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:229)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.index.translog.TranslogException: [cp_text_test][0] failed to create new translog file
	at org.elasticsearch.index.translog.fs.FsTranslog.newTranslog(FsTranslog.java:129)
	at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:266)
	at org.elasticsearch.index.shard.service.InternalIndexShard.start(InternalIndexShard.java:254)
	at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:123)
	at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:179)
	... 3 more
Caused by: java.io.FileNotFoundException: /usr/local/var/elasticsearch/elasticsearch_chrisberkhout/nodes/0/indices/cp_text_test/0/translog/translog-1318328514849 (No such file or directory)
	at java.io.RandomAccessFile.open(Native Method)
	at java.io.RandomAccessFile.<init>(RandomAccessFile.java:216)
	at org.elasticsearch.index.translog.fs.RafReference.<init>(RafReference.java:44)
	at org.elasticsearch.index.translog.fs.FsTranslog.newTranslog(FsTranslog.java:127)
	... 7 more
[2011-10-11 18:21:54,951][WARN ][cluster.action.shard     ] [Zodiak] sending failed shard for [cp_text_test][0], node[kz-d21t7R56bwcSd2E_l_w], [P], s[INITIALIZING], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[cp_text_test][0] failed recovery]; nested: TranslogException[[cp_text_test][0] failed to create new translog file]; nested: FileNotFoundException[/usr/local/var/elasticsearch/elasticsearch_chrisberkhout/nodes/0/indices/cp_text_test/0/translog/translog-1318328514849 (No such file or directory)]; ]]
[2011-10-11 18:21:54,951][WARN ][cluster.action.shard     ] [Zodiak] received shard failed for [cp_text_test][0], node[kz-d21t7R56bwcSd2E_l_w], [P], s[INITIALIZING], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[cp_text_test][0] failed recovery]; nested: TranslogException[[cp_text_test][0] failed to create new translog file]; nested: FileNotFoundException[/usr/local/var/elasticsearch/elasticsearch_chrisberkhout/nodes/0/indices/cp_text_test/0/translog/translog-1318328514849 (No such file or directory)]; ]]
[2011-10-11 18:21:54,973][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] creating index, cause [api], shards [1]/[1], mappings []
[2011-10-11 18:21:55,048][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [document]
[2011-10-11 18:21:55,052][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [person]
[2011-10-11 18:21:55,056][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [source]
[2011-10-11 18:21:55,060][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [tag]
[2011-10-11 18:21:55,074][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] deleting index
[2011-10-11 18:21:55,152][WARN ][indices.cluster          ] [Zodiak] [cp_text_test][0] failed to start shard
org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [cp_text_test][0] failed recovery
	at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:229)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [cp_text_test][0] Failed to open reader on writer
	at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:283)
	at org.elasticsearch.index.shard.service.InternalIndexShard.start(InternalIndexShard.java:254)
	at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:123)
	at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:179)
	... 3 more
Caused by: java.io.FileNotFoundException: /usr/local/var/elasticsearch/elasticsearch_chrisberkhout/nodes/0/indices/cp_text_test/0/index/_checksums-1318328515102 (No such file or directory)
	at java.io.RandomAccessFile.open(Native Method)
	at java.io.RandomAccessFile.<init>(RandomAccessFile.java:216)
	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:441)
	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:306)
	at org.elasticsearch.index.store.support.AbstractStore$StoreDirectory.createOutput(AbstractStore.java:350)
	at org.elasticsearch.index.store.support.AbstractStore.writeChecksums(AbstractStore.java:184)
	at org.elasticsearch.index.store.support.AbstractStore.writeChecksums(AbstractStore.java:171)
	at org.elasticsearch.index.store.support.AbstractStore$StoreDirectory.sync(AbstractStore.java:402)
	at org.apache.lucene.index.SegmentInfos.finishCommit(SegmentInfos.java:924)
	at org.apache.lucene.index.IndexWriter.finishCommit(IndexWriter.java:3516)
	at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3506)
	at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3483)
	at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:264)
	... 6 more
[2011-10-11 18:21:55,164][WARN ][cluster.action.shard     ] [Zodiak] sending failed shard for [cp_text_test][0], node[kz-d21t7R56bwcSd2E_l_w], [P], s[INITIALIZING], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[cp_text_test][0] failed recovery]; nested: EngineCreationFailureException[[cp_text_test][0] Failed to open reader on writer]; nested: FileNotFoundException[/usr/local/var/elasticsearch/elasticsearch_chrisberkhout/nodes/0/indices/cp_text_test/0/index/_checksums-1318328515102 (No such file or directory)]; ]]
[2011-10-11 18:21:55,164][WARN ][cluster.action.shard     ] [Zodiak] received shard failed for [cp_text_test][0], node[kz-d21t7R56bwcSd2E_l_w], [P], s[INITIALIZING], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[cp_text_test][0] failed recovery]; nested: EngineCreationFailureException[[cp_text_test][0] Failed to open reader on writer]; nested: FileNotFoundException[/usr/local/var/elasticsearch/elasticsearch_chrisberkhout/nodes/0/indices/cp_text_test/0/index/_checksums-1318328515102 (No such file or directory)]; ]]
[2011-10-11 18:21:55,165][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] creating index, cause [api], shards [1]/[1], mappings []
[2011-10-11 18:21:55,267][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [document]
[2011-10-11 18:21:55,280][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [person]
[2011-10-11 18:21:55,294][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [source]
[2011-10-11 18:21:55,307][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [tag]
[2011-10-11 18:21:55,321][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] deleting index

# ....

[2011-10-11 18:21:59,158][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] creating index, cause [api], shards [1]/[1], mappings []
[2011-10-11 18:21:59,280][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [document]
[2011-10-11 18:21:59,284][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [person]
[2011-10-11 18:21:59,287][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [source]
[2011-10-11 18:21:59,291][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [tag]
[2011-10-11 18:21:59,299][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] deleting index
[2011-10-11 18:21:59,315][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] creating index, cause [api], shards [1]/[1], mappings []
[2011-10-11 18:21:59,329][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [document]
[2011-10-11 18:21:59,333][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [person]
[2011-10-11 18:21:59,337][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [source]
[2011-10-11 18:21:59,340][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [tag]
[2011-10-11 18:21:59,349][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] deleting index
[2011-10-11 18:21:59,355][WARN ][indices.cluster          ] [Zodiak] [cp_text_test][0] failed to start shard
org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [cp_text_test][0] failed recovery
	at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:229)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [cp_text_test][0] Failed to open reader on writer
	at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:283)
	at org.elasticsearch.index.shard.service.InternalIndexShard.start(InternalIndexShard.java:254)
	at org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:123)
	at org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:179)
	... 3 more
Caused by: java.io.FileNotFoundException: /usr/local/var/elasticsearch/elasticsearch_chrisberkhout/nodes/0/indices/cp_text_test/0/index/segments_1 (No such file or directory)
	at java.io.RandomAccessFile.open(Native Method)
	at java.io.RandomAccessFile.<init>(RandomAccessFile.java:216)
	at org.apache.lucene.store.SimpleFSDirectory$SimpleFSIndexInput$Descriptor.<init>(SimpleFSDirectory.java:69)
	at org.apache.lucene.store.SimpleFSDirectory$SimpleFSIndexInput.<init>(SimpleFSDirectory.java:90)
	at org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.<init>(NIOFSDirectory.java:91)
	at org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:78)
	at org.apache.lucene.store.FSDirectory.openInput(FSDirectory.java:345)
	at org.elasticsearch.index.store.support.AbstractStore$StoreDirectory.openInput(AbstractStore.java:360)
	at org.apache.lucene.index.SegmentInfos.read(SegmentInfos.java:265)
	at org.apache.lucene.index.SegmentInfos$1.doBody(SegmentInfos.java:363)
	at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:754)
	at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:593)
	at org.apache.lucene.index.SegmentInfos.read(SegmentInfos.java:359)
	at org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:269)
	... 6 more
[2011-10-11 18:21:59,356][WARN ][cluster.action.shard     ] [Zodiak] sending failed shard for [cp_text_test][0], node[kz-d21t7R56bwcSd2E_l_w], [P], s[INITIALIZING], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[cp_text_test][0] failed recovery]; nested: EngineCreationFailureException[[cp_text_test][0] Failed to open reader on writer]; nested: FileNotFoundException[/usr/local/var/elasticsearch/elasticsearch_chrisberkhout/nodes/0/indices/cp_text_test/0/index/segments_1 (No such file or directory)]; ]]
[2011-10-11 18:21:59,356][WARN ][cluster.action.shard     ] [Zodiak] received shard failed for [cp_text_test][0], node[kz-d21t7R56bwcSd2E_l_w], [P], s[INITIALIZING], reason [Failed to start shard, message [IndexShardGatewayRecoveryException[[cp_text_test][0] failed recovery]; nested: EngineCreationFailureException[[cp_text_test][0] Failed to open reader on writer]; nested: FileNotFoundException[/usr/local/var/elasticsearch/elasticsearch_chrisberkhout/nodes/0/indices/cp_text_test/0/index/segments_1 (No such file or directory)]; ]]
[2011-10-11 18:21:59,379][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] creating index, cause [api], shards [1]/[1], mappings []
[2011-10-11 18:21:59,392][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [document]
[2011-10-11 18:21:59,397][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [person]
[2011-10-11 18:21:59,401][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [source]
[2011-10-11 18:21:59,404][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] create_mapping [tag]
[2011-10-11 18:21:59,413][INFO ][cluster.metadata         ] [Zodiak] [cp_text_test] deleting index

elasticsearch-tire.log

# 2011-10-11 18:21:56:065 [DELETE] ("cp_text_test")
#
curl -X DELETE "http://localhost:9200/cp_text_test"

# 2011-10-11 18:21:56:065 [200]

# 2011-10-11 18:21:56:094 [CREATE] ("cp_text_test")
#
curl -X POST "http://localhost:9200/cp_text_test" -d '{"settings":{"number_of_shards":1,"analysis":{"analyzer":{"default":{"type":"english"}}}}}'

# 2011-10-11 18:21:56:094 [200]

# 2011-10-11 18:21:56:112 [_refresh] ("cp_text_test")
#
curl -X POST "http://localhost:9200/cp_text_test/_refresh"

# 2011-10-11 18:21:56:112 [200]

# 2011-10-11 18:21:56:119 [_search] (["cp_text_test"])
#
curl -X GET "http://localhost:9200/cp_text_test/_search?pretty=true" -d '{"sort":[{"date":"desc"}],"facets":{"current_tags":{"terms":{"field":"tag_ids","size":999999999,"all_terms":false,"exclude":[]}},"current_authors":{"terms":{"field":"author_ids","size":999999999,"all_terms":false,"exclude":[]}},"current_sources":{"terms":{"field":"source_ids","size":999999999,"all_terms":false,"exclude":[]}}},"filter":{"type":{"value":"document"}},"size":50,"from":0}'

# 2011-10-11 18:21:56:119 [500]

# 2011-10-11 18:21:56:143 [DELETE] ("cp_text_test")
#
curl -X DELETE "http://localhost:9200/cp_text_test"

# 2011-10-11 18:21:56:143 [200]

# ....

# 2011-10-11 18:21:58:843 [DELETE] ("cp_text_test")
#
curl -X DELETE "http://localhost:9200/cp_text_test"

# 2011-10-11 18:21:58:843 [200]

# 2011-10-11 18:21:58:863 [CREATE] ("cp_text_test")
#
curl -X POST "http://localhost:9200/cp_text_test" -d '{"settings":{"number_of_shards":1,"analysis":{"analyzer":{"default":{"type":"english"}}}}}'

# 2011-10-11 18:21:58:863 [200]

# 2011-10-11 18:21:58:884 [_refresh] ("cp_text_test")
#
curl -X POST "http://localhost:9200/cp_text_test/_refresh"

# 2011-10-11 18:21:58:884 [200]

# 2011-10-11 18:21:58:888 [_search] (["cp_text_test"])
#
curl -X GET "http://localhost:9200/cp_text_test/_search?pretty=true" -d '{"query":{"match_all":{}},"filter":{"type":{"value":"tag"}},"size":15}'

# 2011-10-11 18:21:58:889 [500]

# 2011-10-11 18:21:58:945 [DELETE] ("cp_text_test")
#
curl -X DELETE "http://localhost:9200/cp_text_test"

# 2011-10-11 18:21:58:945 [200]