es_failed_shard.log

[2013-06-12 00:19:48,237][WARN ][action.index             ] [Lang, Cassie] Failed to perform index on replica [producteev][2]
org.elasticsearch.transport.RemoteTransportException: [Centennial][inet[/10.188.188.42:9300]][index/replica]
Caused by: org.elasticsearch.index.engine.IndexFailedEngineException: [producteev][2] Index failed for [tasks#51b7bebd7c07cd2074000001]
        at org.elasticsearch.index.engine.robin.RobinEngine.index(RobinEngine.java:493)
        at org.elasticsearch.index.shard.service.InternalIndexShard.index(InternalIndexShard.java:338)
        at org.elasticsearch.action.index.TransportIndexAction.shardOperationOnReplica(TransportIndexAction.java:261)
        at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$ReplicaOperationTransportHandler.messageReceived(TransportShardReplicationOperationAction.java:254)
        at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$ReplicaOperationTransportHandler.messageReceived(TransportShardReplicationOperationAction.java:240)
        at org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:268)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:722)
Caused by: java.lang.OutOfMemoryError: Java heap space
[2013-06-12 00:19:48,238][WARN ][cluster.action.shard     ] [Lang, Cassie] sending failed shard for [producteev][2], node[RvefrhO2T92o48-fTIAi7Q], [R], s[STARTED], reason [Failed to perform [index] on replica, message [RemoteTransportException[[Centennial][inet[/10.188.188.42:9300]][index/replica]]; nested: IndexFailedEngineException[[producteev][2] Index failed for [tasks#51b7bebd7c07cd2074000001]]; nested: OutOfMemoryError[Java heap space]; ]]
[2013-06-12 00:22:57,629][INFO ][discovery.zen            ] [Lang, Cassie] master_left [[Frost, Adrienne][CXMdYSDTShCtJOGFg7tbKQ][inet[/10.188.190.51:9300]]], reason [shut_down]
[2013-06-12 00:22:57,629][INFO ][cluster.service          ] [Lang, Cassie] master {new [Sun Girl][LOd1c7pjRqChJf7og5rMjQ][inet[/10.188.154.103:9300]], previous [Frost, Adrienne][CXMdYSDTShCtJOGFg7tbKQ][inet[/10.188.190.51:9300]]}, removed {[Frost, Adrienne][CXMdYSDTShCtJOGFg7tbKQ][inet[/10.188.190.51:9300]],}, reason: zen-disco-master_failed ([Frost, Adrienne][CXMdYSDTShCtJOGFg7tbKQ][inet[/10.188.190.51:9300]])
[2013-06-12 00:23:04,979][INFO ][cluster.service          ] [Lang, Cassie] added {[Wizard][msUHCUY8R4WmXhMCZkBZlg][inet[/10.188.190.51:9300]],}, reason: zen-disco-receive(from master [[Sun Girl][LOd1c7pjRqChJf7og5rMjQ][inet[/10.188.154.103:9300]]])
[2013-06-12 00:24:21,753][INFO ][cluster.service          ] [Lang, Cassie] removed {[Wizard][msUHCUY8R4WmXhMCZkBZlg][inet[/10.188.190.51:9300]],}, reason: zen-disco-receive(from master [[Sun Girl][LOd1c7pjRqChJf7og5rMjQ][inet[/10.188.154.103:9300]]])
[2013-06-12 00:24:25,745][INFO ][cluster.service          ] [Lang, Cassie] added {[Paul Norbert Ebersol][Al9dqlaTQZe2XyQ6PB-WKg][inet[/10.188.190.51:9300]],}, reason: zen-disco-receive(from master [[Sun Girl][LOd1c7pjRqChJf7og5rMjQ][inet[/10.188.154.103:9300]]])
[2013-06-12 00:27:20,505][INFO ][cluster.service          ] [Lang, Cassie] removed {[Centennial][RvefrhO2T92o48-fTIAi7Q][inet[/10.188.188.42:9300]],}, reason: zen-disco-receive(from master [[Sun Girl][LOd1c7pjRqChJf7og5rMjQ][inet[/10.188.154.103:9300]]])
[2013-06-12 00:27:31,286][INFO ][cluster.service          ] [Lang, Cassie] master {new [Centennial][RvefrhO2T92o48-fTIAi7Q][inet[/10.188.188.42:9300]], previous [Sun Girl][LOd1c7pjRqChJf7og5rMjQ][inet[/10.188.154.103:9300]]}, removed {[Sun Girl][LOd1c7pjRqChJf7og5rMjQ][inet[/10.188.154.103:9300]],[Paul Norbert Ebersol][Al9dqlaTQZe2XyQ6PB-WKg][inet[/10.188.190.51:9300]],}, added {[Centennial][RvefrhO2T92o48-fTIAi7Q][inet[/10.188.188.42:9300]],}, reason: zen-disco-receive(from master [[Centennial][RvefrhO2T92o48-fTIAi7Q][inet[/10.188.188.42:9300]]])
[2013-06-12 00:27:31,287][WARN ][index.shard.service      ] [Lang, Cassie] [producteev][4] suspect illegal state: trying to move shard from primary mode to replica mode