Exception when reaching maximum number of memory mapped files.

elasticsearch.log

To see more:
- http://jprante.github.io/applications/2012/07/26/Mmap-with-Lucene.html
- http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html

The store type in ES changed from 0.90.9 to 1.2 to match the Lucene defaults, which means that now its mmapfs by default.
You are probably running into the OS limits of mmap counts.
You might need to run something like this:
sysctl -q -w vm.max_map_count=262144

129450 [2014-06-12 09:32:14,061][DEBUG][action.admin.cluster.stats] [XXXXXXXX] failed to execute on node [ABl2CA6_SuGDxdmXnqMgFA]
129451 org.elasticsearch.index.engine.EngineClosedException: [logstash-prd-XXX-2014.06.12][3] CurrentState[CLOSED]
129452    at org.elasticsearch.index.engine.internal.InternalEngine.ensureOpen(InternalEngine.java:851)
129453    at org.elasticsearch.index.engine.internal.InternalEngine.acquireSearcher(InternalEngine.java:660)
129454    at org.elasticsearch.index.shard.service.InternalIndexShard.acquireSearcher(InternalIndexShard.java:648)
129455    at org.elasticsearch.index.shard.service.InternalIndexShard.acquireSearcher(InternalIndexShard.java:642)
129456    at org.elasticsearch.index.shard.service.InternalIndexShard.docStats(InternalIndexShard.java:495)
129457    at org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:127)
129458    at org.elasticsearch.action.admin.indices.stats.ShardStats.<init>(ShardStats.java:49)
129459    at org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:130)
129460    at org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:54)
129461    at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction$2.run(TransportNodesOperationAction.java:146)
129462    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
129463    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
129464    at java.lang.Thread.run(Thread.java:745)
129465 Caused by: java.io.IOException: Map failed
129466    at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:888)
129467    at org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:279)
129468    at org.apache.lucene.store.MMapDirectory$MMapIndexInput.<init>(MMapDirectory.java:225)
129469    at org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:194)
129470    at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:80)
129471    at org.apache.lucene.store.Directory$1.<init>(Directory.java:227)
129472    at org.apache.lucene.store.Directory.createSlicer(Directory.java:226)
129473    at org.elasticsearch.index.store.Store$StoreDirectory.createSlicer(Store.java:547)
129474    at org.apache.lucene.store.CompoundFileDirectory.<init>(CompoundFileDirectory.java:103)
129475    at org.apache.lucene.index.SegmentReader.readFieldInfos(SegmentReader.java:209)
129476    at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:100)
129477    at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:142)
129478    at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:236)
129479    at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:99)
129480    at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:385)
129481    at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:287)
129482    at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
129483    at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:252)
129484    at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:171)
129485    at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:118)
129486    at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
129487    at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
129488    at org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
129489    at org.elasticsearch.index.engine.internal.InternalEngine.refresh(InternalEngine.java:715)
129490    at org.elasticsearch.index.shard.service.InternalIndexShard.refresh(InternalIndexShard.java:479)
129491    at org.elasticsearch.index.shard.service.InternalIndexShard$EngineRefresher$1.run(InternalIndexShard.java:928)
129492    ... 3 more