splitbrain logfiles

splitbrain-isolatednode.log

[2012-08-01 14:42:13,257][INFO ][node                     ] [isolatednode] {0.20.0.Beta1-SNAPSHOT}[3236]: initializing ...
[2012-08-01 14:42:13,262][INFO ][plugins                  ] [isolatednode] loaded [], sites []
[2012-08-01 14:42:14,205][DEBUG][discovery.zen.ping.multicast] [isolatednode] using group [224.2.2.4], with port [54328], ttl [3], and address [null]
[2012-08-01 14:42:14,207][DEBUG][discovery.zen.ping.unicast] [isolatednode] using initial hosts [], with concurrent_connects [10]
[2012-08-01 14:42:14,208][DEBUG][discovery.zen            ] [isolatednode] using ping.timeout [3s], master_election.filter_client [true], master_election.filter_data [false]
[2012-08-01 14:42:14,212][DEBUG][discovery.zen.elect      ] [isolatednode] using minimum_master_nodes [2]
[2012-08-01 14:42:14,213][DEBUG][discovery.zen.fd         ] [isolatednode] [master] uses ping_interval [1s], ping_timeout [3s], ping_retries [2]
[2012-08-01 14:42:14,215][DEBUG][discovery.zen.fd         ] [isolatednode] [node  ] uses ping_interval [1s], ping_timeout [3s], ping_retries [2]
[2012-08-01 14:42:14,984][DEBUG][cluster.routing.allocation.decider] [isolatednode] using node_concurrent_recoveries [2], node_initial_primaries_recoveries [4]
[2012-08-01 14:42:14,984][DEBUG][cluster.routing.allocation.decider] [isolatednode] using [cluster.routing.allocation.allow_rebalance] with [indices_all_active]
[2012-08-01 14:42:14,985][DEBUG][cluster.routing.allocation.decider] [isolatednode] using [cluster_concurrent_rebalance] with [2]
[2012-08-01 14:42:15,133][INFO ][node                     ] [isolatednode] {0.20.0.Beta1-SNAPSHOT}[3236]: initialized
[2012-08-01 14:42:15,134][INFO ][node                     ] [isolatednode] {0.20.0.Beta1-SNAPSHOT}[3236]: starting ...
[2012-08-01 14:42:15,218][INFO ][transport                ] [isolatednode] bound_address {inet[/0:0:0:0:0:0:0:0%0:9300]}, publish_address {inet[/192.168.7.130:9300]}
[2012-08-01 14:42:15,229][TRACE][discovery                ] [isolatednode] waiting for 30s for the initial state to be set by the discovery
[2012-08-01 14:42:15,247][TRACE][discovery.zen.ping.multicast] [isolatednode] [1] sending ping request
[2012-08-01 14:42:15,285][TRACE][discovery.zen.ping.multicast] [isolatednode] [1] received ping_response{target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]], cluster_name[splitbrain]}
[2012-08-01 14:42:15,285][TRACE][discovery.zen.ping.multicast] [isolatednode] [1] received ping_response{target [[node2][zzznode2][inet[/192.168.7.132:9301]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]], cluster_name[splitbrain]}
[2012-08-01 14:42:16,749][TRACE][discovery.zen.ping.multicast] [isolatednode] [1] sending ping request
[2012-08-01 14:42:16,751][TRACE][discovery.zen.ping.multicast] [isolatednode] [1] received ping_response{target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]], cluster_name[splitbrain]}
[2012-08-01 14:42:16,751][TRACE][discovery.zen.ping.multicast] [isolatednode] [1] received ping_response{target [[node2][zzznode2][inet[/192.168.7.132:9301]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]], cluster_name[splitbrain]}
[2012-08-01 14:42:18,257][TRACE][discovery.zen            ] [isolatednode] full ping responses:
	--> target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]]
	--> target [[node2][zzznode2][inet[/192.168.7.132:9301]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]]
[2012-08-01 14:42:18,257][DEBUG][discovery.zen            ] [isolatednode] filtered ping responses: (filter_client[true], filter_data[false])
	--> target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]]
	--> target [[node2][zzznode2][inet[/192.168.7.132:9301]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]]
[2012-08-01 14:42:18,285][DEBUG][discovery.zen.fd         ] [isolatednode] [master] starting fault detection against master [[node1][zzznode1][inet[/192.168.7.132:9300]]], reason [initial_join]
[2012-08-01 14:42:18,291][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:18,291][DEBUG][cluster.service          ] [isolatednode] got first state from fresh master [zzznode1]
[2012-08-01 14:42:18,293][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [4], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
routing_nodes:
---- unassigned

[2012-08-01 14:42:18,295][INFO ][cluster.service          ] [isolatednode] detected_master [node1][zzznode1][inet[/192.168.7.132:9300]], added {[node2][zzznode2][inet[/192.168.7.132:9301]],[node1][zzznode1][inet[/192.168.7.132:9300]],}, reason: zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])
[2012-08-01 14:42:18,400][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:18,400][TRACE][discovery                ] [isolatednode] initial state set from discovery
[2012-08-01 14:42:18,400][INFO ][discovery                ] [isolatednode] splitbrain/aaanode3
[2012-08-01 14:42:18,404][INFO ][http                     ] [isolatednode] bound_address {inet[/0:0:0:0:0:0:0:0%0:9200]}, publish_address {inet[/192.168.7.130:9200]}
[2012-08-01 14:42:18,405][INFO ][node                     ] [isolatednode] {0.20.0.Beta1-SNAPSHOT}[3236]: started
[2012-08-01 14:42:36,144][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:36,145][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [5], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][0], node[null], [R], s[UNASSIGNED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[INITIALIZING]
--------[test1][1], node[null], [R], s[UNASSIGNED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
--------[test1][2], node[null], [R], s[UNASSIGNED]
----shard_id [test1][3]
--------[test1][3], node[zzznode1], [P], s[INITIALIZING]
--------[test1][3], node[null], [R], s[UNASSIGNED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[null], [R], s[UNASSIGNED]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
-----node_id[zzznode2][V]
--------[test1][1], node[zzznode2], [P], s[INITIALIZING]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][3], node[zzznode1], [P], s[INITIALIZING]
---- unassigned
--------[test1][0], node[null], [R], s[UNASSIGNED]
--------[test1][1], node[null], [R], s[UNASSIGNED]
--------[test1][2], node[null], [R], s[UNASSIGNED]
--------[test1][3], node[null], [R], s[UNASSIGNED]
--------[test1][4], node[null], [R], s[UNASSIGNED]

[2012-08-01 14:42:36,501][DEBUG][cluster.action.shard     ] [isolatednode] sending shard started for [test1][0], node[aaanode3], [P], s[INITIALIZING], reason [after recovery from gateway]
[2012-08-01 14:42:36,503][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:36,504][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:36,504][DEBUG][cluster.action.shard     ] [isolatednode] sending shard started for [test1][4], node[aaanode3], [P], s[INITIALIZING], reason [after recovery from gateway]
[2012-08-01 14:42:36,504][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [6], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][0], node[null], [R], s[UNASSIGNED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[INITIALIZING]
--------[test1][1], node[null], [R], s[UNASSIGNED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
--------[test1][2], node[null], [R], s[UNASSIGNED]
----shard_id [test1][3]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][3], node[null], [R], s[UNASSIGNED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[null], [R], s[UNASSIGNED]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
-----node_id[zzznode2][V]
--------[test1][1], node[zzznode2], [P], s[INITIALIZING]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][3], node[zzznode1], [P], s[STARTED]
---- unassigned
--------[test1][0], node[null], [R], s[UNASSIGNED]
--------[test1][1], node[null], [R], s[UNASSIGNED]
--------[test1][2], node[null], [R], s[UNASSIGNED]
--------[test1][3], node[null], [R], s[UNASSIGNED]
--------[test1][4], node[null], [R], s[UNASSIGNED]

[2012-08-01 14:42:36,505][DEBUG][cluster.action.shard     ] [isolatednode] sending shard started for [test1][0], node[aaanode3], [P], s[INITIALIZING], reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]
[2012-08-01 14:42:36,505][DEBUG][cluster.action.shard     ] [isolatednode] sending shard started for [test1][4], node[aaanode3], [P], s[INITIALIZING], reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]
[2012-08-01 14:42:36,506][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:36,510][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:36,511][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [7], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][0], node[null], [R], s[UNASSIGNED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
--------[test1][2], node[null], [R], s[UNASSIGNED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[INITIALIZING]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[null], [R], s[UNASSIGNED]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
-----node_id[zzznode2][V]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
--------[test1][3], node[zzznode2], [R], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
--------[test1][3], node[zzznode1], [P], s[STARTED]
---- unassigned
--------[test1][0], node[null], [R], s[UNASSIGNED]
--------[test1][2], node[null], [R], s[UNASSIGNED]
--------[test1][4], node[null], [R], s[UNASSIGNED]

[2012-08-01 14:42:36,512][DEBUG][cluster.action.shard     ] [isolatednode] sending shard started for [test1][0], node[aaanode3], [P], s[INITIALIZING], reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]
[2012-08-01 14:42:36,512][DEBUG][cluster.action.shard     ] [isolatednode] sending shard started for [test1][4], node[aaanode3], [P], s[INITIALIZING], reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]
[2012-08-01 14:42:36,513][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:36,561][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:36,562][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [8], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[INITIALIZING]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[INITIALIZING]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[INITIALIZING]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]
---- unassigned

[2012-08-01 14:42:36,596][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:36,597][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:36,598][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [9], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[INITIALIZING]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[INITIALIZING]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]
---- unassigned

[2012-08-01 14:42:36,599][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:36,617][DEBUG][cluster.action.shard     ] [isolatednode] sending shard started for [test1][2], node[aaanode3], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:42:36,622][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:36,623][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [10], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]
---- unassigned

[2012-08-01 14:42:36,625][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:36,635][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:36,636][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [11], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]
---- unassigned

[2012-08-01 14:42:36,638][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:36,647][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:36,647][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [12], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]
---- unassigned

[2012-08-01 14:42:36,649][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:49,337][TRACE][discovery.zen.fd         ] [isolatednode] [master] failed to ping [[node1][zzznode1][inet[/192.168.7.132:9300]]], retry [1] out of [2]
org.elasticsearch.transport.ReceiveTimeoutTransportException: [node1][inet[/192.168.7.132:9300]][discovery/zen/fd/masterPing] request_id [81] timed out after [3001ms]
	at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:347)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:42:52,341][TRACE][discovery.zen.fd         ] [isolatednode] [master] failed to ping [[node1][zzznode1][inet[/192.168.7.132:9300]]], retry [2] out of [2]
org.elasticsearch.transport.ReceiveTimeoutTransportException: [node1][inet[/192.168.7.132:9300]][discovery/zen/fd/masterPing] request_id [82] timed out after [3001ms]
	at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:347)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:42:52,341][DEBUG][discovery.zen.fd         ] [isolatednode] [master] failed to ping [[node1][zzznode1][inet[/192.168.7.132:9300]]], tried [2] times, each with maximum [3s] timeout
[2012-08-01 14:42:52,342][DEBUG][discovery.zen.fd         ] [isolatednode] [master] stopping fault detection against master [[node1][zzznode1][inet[/192.168.7.132:9300]]], reason [master failure, failed to ping, tried [2] times, each with  maximum [3s] timeout]
[2012-08-01 14:42:52,342][INFO ][discovery.zen            ] [isolatednode] master_left [[node1][zzznode1][inet[/192.168.7.132:9300]]], reason [failed to ping, tried [2] times, each with  maximum [3s] timeout]
[2012-08-01 14:42:52,343][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-master_failed ([node1][zzznode1][inet[/192.168.7.132:9300]])]: execute
[2012-08-01 14:42:52,343][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [13], source [zen-disco-master_failed ([node1][zzznode1][inet[/192.168.7.132:9300]])]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local, master
   [node2][zzznode2][inet[/192.168.7.132:9301]]
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][X]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]
---- unassigned

[2012-08-01 14:42:52,343][INFO ][cluster.service          ] [isolatednode] master {new [isolatednode][aaanode3][inet[/192.168.7.130:9300]], previous [node1][zzznode1][inet[/192.168.7.132:9300]]}, removed {[node1][zzznode1][inet[/192.168.7.132:9300]],}, reason: zen-disco-master_failed ([node1][zzznode1][inet[/192.168.7.132:9300]])
[2012-08-01 14:42:52,348][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-master_failed ([node1][zzznode1][inet[/192.168.7.132:9300]])]: done applying updated cluster_state
[2012-08-01 14:42:52,348][DEBUG][cluster.service          ] [isolatednode] processing [routing-table-updater]: execute
[2012-08-01 14:42:56,347][TRACE][discovery.zen.fd         ] [isolatednode] [node  ] failed to ping [[node2][zzznode2][inet[/192.168.7.132:9301]]], retry [1] out of [2]
org.elasticsearch.transport.ReceiveTimeoutTransportException: [node2][inet[/192.168.7.132:9301]][discovery/zen/fd/ping] request_id [85] timed out after [3001ms]
	at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:347)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:42:59,350][TRACE][discovery.zen.fd         ] [isolatednode] [node  ] failed to ping [[node2][zzznode2][inet[/192.168.7.132:9301]]], retry [2] out of [2]
org.elasticsearch.transport.ReceiveTimeoutTransportException: [node2][inet[/192.168.7.132:9301]][discovery/zen/fd/ping] request_id [86] timed out after [3001ms]
	at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:347)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:42:59,350][DEBUG][discovery.zen.fd         ] [isolatednode] [node  ] failed to ping [[node2][zzznode2][inet[/192.168.7.132:9301]]], tried [2] times, each with  maximum [3s] timeout
[2012-08-01 14:43:47,288][WARN ][transport.netty          ] [isolatednode] Exception caught on netty layer [[id: 0x718df055, /192.168.7.130:52781 => /192.168.7.132:9301]]
java.io.IOException: Operation timed out
	at sun.nio.ch.FileDispatcher.read0(Native Method)
	at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:21)
	at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:198)
	at sun.nio.ch.IOUtil.read(IOUtil.java:166)
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:245)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:63)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:43:47,288][WARN ][transport.netty          ] [isolatednode] Exception caught on netty layer [[id: 0x61735602, /192.168.7.130:52779 => /192.168.7.132:9301]]
java.io.IOException: Operation timed out
	at sun.nio.ch.FileDispatcher.read0(Native Method)
	at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:21)
	at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:198)
	at sun.nio.ch.IOUtil.read(IOUtil.java:166)
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:245)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:63)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:43:47,291][DEBUG][discovery.zen.publish    ] [isolatednode] failed to send cluster state to [[node2][zzznode2][inet[/192.168.7.132:9301]]], should be detected as failed soon...
org.elasticsearch.transport.NodeDisconnectedException: [node2][inet[/192.168.7.132:9301]][discovery/zen/publish] disconnected
[2012-08-01 14:43:47,294][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [14], source [routing-table-updater]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local, master
   [node2][zzznode2][inet[/192.168.7.132:9301]]
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[aaanode3], [R], s[INITIALIZING]
--------[test1][1], node[zzznode2], [P], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[aaanode3], [R], s[INITIALIZING]
--------[test1][3], node[zzznode2], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[INITIALIZING]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[INITIALIZING]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][3], node[aaanode3], [R], s[INITIALIZING]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [P], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[INITIALIZING]
---- unassigned

[2012-08-01 14:43:47,294][DEBUG][discovery.zen.publish    ] [isolatednode] failed to send cluster state to [[node2][zzznode2][inet[/192.168.7.132:9301]]], should be detected as failed soon...
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][discovery/zen/publish]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.discovery.zen.publish.PublishClusterStateAction.publish(PublishClusterStateAction.java:89)
	at org.elasticsearch.discovery.zen.ZenDiscovery.publish(ZenDiscovery.java:268)
	at org.elasticsearch.discovery.DiscoveryService.publish(DiscoveryService.java:115)
	at org.elasticsearch.cluster.service.InternalClusterService$2.run(InternalClusterService.java:293)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 7 more
[2012-08-01 14:43:47,355][DEBUG][cluster.service          ] [isolatednode] processing [routing-table-updater]: done applying updated cluster_state
[2012-08-01 14:43:47,356][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-node_failed([node2][zzznode2][inet[/192.168.7.132:9301]]), reason failed to ping, tried [2] times, each with maximum [3s] timeout]: execute
[2012-08-01 14:43:47,356][WARN ][discovery.zen            ] [isolatednode] not enough master nodes, current nodes: {[isolatednode][aaanode3][inet[/192.168.7.130:9300]],}
[2012-08-01 14:43:47,356][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [14], source [zen-disco-node_failed([node2][zzznode2][inet[/192.168.7.132:9301]]), reason failed to ping, tried [2] times, each with maximum [3s] timeout]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local
routing_table:
routing_nodes:
---- unassigned

[2012-08-01 14:43:47,357][INFO ][cluster.service          ] [isolatednode] removed {[node2][zzznode2][inet[/192.168.7.132:9301]],}, reason: zen-disco-node_failed([node2][zzznode2][inet[/192.168.7.132:9301]]), reason failed to ping, tried [2] times, each with maximum [3s] timeout
[2012-08-01 14:43:47,357][TRACE][discovery.zen.ping.multicast] [isolatednode] [2] sending ping request
[2012-08-01 14:43:47,379][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-node_failed([node2][zzznode2][inet[/192.168.7.132:9301]]), reason failed to ping, tried [2] times, each with maximum [3s] timeout]: done applying updated cluster_state
[2012-08-01 14:43:48,858][TRACE][discovery.zen.ping.multicast] [isolatednode] [2] sending ping request
[2012-08-01 14:43:50,358][TRACE][discovery.zen            ] [isolatednode] full ping responses: {none}
[2012-08-01 14:43:50,359][DEBUG][discovery.zen            ] [isolatednode] filtered ping responses: (filter_client[true], filter_data[false]) {none}
[2012-08-01 14:43:50,359][TRACE][discovery.zen.ping.multicast] [isolatednode] [3] sending ping request
[2012-08-01 14:43:51,860][TRACE][discovery.zen.ping.multicast] [isolatednode] [3] sending ping request
[2012-08-01 14:43:53,362][TRACE][discovery.zen            ] [isolatednode] full ping responses: {none}
[2012-08-01 14:43:53,362][DEBUG][discovery.zen            ] [isolatednode] filtered ping responses: (filter_client[true], filter_data[false]) {none}
[2012-08-01 14:43:53,363][TRACE][discovery.zen.ping.multicast] [isolatednode] [4] sending ping request
[2012-08-01 14:43:54,863][TRACE][discovery.zen.ping.multicast] [isolatednode] [4] sending ping request
[2012-08-01 14:43:56,364][TRACE][discovery.zen            ] [isolatednode] full ping responses: {none}
[2012-08-01 14:43:56,364][DEBUG][discovery.zen            ] [isolatednode] filtered ping responses: (filter_client[true], filter_data[false]) {none}
[2012-08-01 14:43:56,364][TRACE][discovery.zen.ping.multicast] [isolatednode] [5] sending ping request
[2012-08-01 14:43:57,866][TRACE][discovery.zen.ping.multicast] [isolatednode] [5] sending ping request
[2012-08-01 14:43:59,367][TRACE][discovery.zen            ] [isolatednode] full ping responses: {none}
[2012-08-01 14:43:59,367][DEBUG][discovery.zen            ] [isolatednode] filtered ping responses: (filter_client[true], filter_data[false]) {none}
[2012-08-01 14:43:59,368][TRACE][discovery.zen.ping.multicast] [isolatednode] [6] sending ping request
[2012-08-01 14:44:00,869][TRACE][discovery.zen.ping.multicast] [isolatednode] [6] sending ping request
[2012-08-01 14:44:02,370][TRACE][discovery.zen            ] [isolatednode] full ping responses: {none}
[2012-08-01 14:44:02,370][DEBUG][discovery.zen            ] [isolatednode] filtered ping responses: (filter_client[true], filter_data[false]) {none}
[2012-08-01 14:44:02,370][TRACE][discovery.zen.ping.multicast] [isolatednode] [7] sending ping request
[2012-08-01 14:44:03,872][TRACE][discovery.zen.ping.multicast] [isolatednode] [7] sending ping request
[2012-08-01 14:44:05,372][TRACE][discovery.zen            ] [isolatednode] full ping responses: {none}
[2012-08-01 14:44:05,372][DEBUG][discovery.zen            ] [isolatednode] filtered ping responses: (filter_client[true], filter_data[false]) {none}
[2012-08-01 14:44:05,373][TRACE][discovery.zen.ping.multicast] [isolatednode] [8] sending ping request
[2012-08-01 14:44:06,873][TRACE][discovery.zen.ping.multicast] [isolatednode] [8] sending ping request
[2012-08-01 14:44:08,375][TRACE][discovery.zen            ] [isolatednode] full ping responses: {none}
[2012-08-01 14:44:08,375][DEBUG][discovery.zen            ] [isolatednode] filtered ping responses: (filter_client[true], filter_data[false]) {none}
[2012-08-01 14:44:08,375][TRACE][discovery.zen.ping.multicast] [isolatednode] [9] sending ping request
[2012-08-01 14:44:09,876][TRACE][discovery.zen.ping.multicast] [isolatednode] [9] sending ping request
[2012-08-01 14:44:11,377][TRACE][discovery.zen            ] [isolatednode] full ping responses: {none}
[2012-08-01 14:44:11,377][DEBUG][discovery.zen            ] [isolatednode] filtered ping responses: (filter_client[true], filter_data[false]) {none}
[2012-08-01 14:44:11,377][TRACE][discovery.zen.ping.multicast] [isolatednode] [10] sending ping request
[2012-08-01 14:44:12,879][TRACE][discovery.zen.ping.multicast] [isolatednode] [10] sending ping request
[2012-08-01 14:44:14,380][TRACE][discovery.zen            ] [isolatednode] full ping responses: {none}
[2012-08-01 14:44:14,380][DEBUG][discovery.zen            ] [isolatednode] filtered ping responses: (filter_client[true], filter_data[false]) {none}
[2012-08-01 14:44:14,380][TRACE][discovery.zen.ping.multicast] [isolatednode] [11] sending ping request
[2012-08-01 14:44:15,882][TRACE][discovery.zen.ping.multicast] [isolatednode] [11] sending ping request
[2012-08-01 14:44:17,383][TRACE][discovery.zen            ] [isolatednode] full ping responses: {none}
[2012-08-01 14:44:17,383][DEBUG][discovery.zen            ] [isolatednode] filtered ping responses: (filter_client[true], filter_data[false]) {none}
[2012-08-01 14:44:17,383][TRACE][discovery.zen.ping.multicast] [isolatednode] [12] sending ping request
[2012-08-01 14:44:18,884][TRACE][discovery.zen.ping.multicast] [isolatednode] [12] sending ping request
[2012-08-01 14:44:20,385][TRACE][discovery.zen            ] [isolatednode] full ping responses: {none}
[2012-08-01 14:44:20,385][DEBUG][discovery.zen            ] [isolatednode] filtered ping responses: (filter_client[true], filter_data[false]) {none}
[2012-08-01 14:44:20,385][TRACE][discovery.zen.ping.multicast] [isolatednode] [13] sending ping request
[2012-08-01 14:44:21,886][TRACE][discovery.zen.ping.multicast] [isolatednode] [13] sending ping request
[2012-08-01 14:44:23,387][TRACE][discovery.zen            ] [isolatednode] full ping responses: {none}
[2012-08-01 14:44:23,387][DEBUG][discovery.zen            ] [isolatednode] filtered ping responses: (filter_client[true], filter_data[false]) {none}
[2012-08-01 14:44:23,388][TRACE][discovery.zen.ping.multicast] [isolatednode] [14] sending ping request
[2012-08-01 14:44:24,890][TRACE][discovery.zen.ping.multicast] [isolatednode] [14] sending ping request
[2012-08-01 14:44:26,391][TRACE][discovery.zen            ] [isolatednode] full ping responses: {none}
[2012-08-01 14:44:26,391][DEBUG][discovery.zen            ] [isolatednode] filtered ping responses: (filter_client[true], filter_data[false]) {none}
[2012-08-01 14:44:26,391][TRACE][discovery.zen.ping.multicast] [isolatednode] [15] sending ping request
[2012-08-01 14:44:27,893][TRACE][discovery.zen.ping.multicast] [isolatednode] [15] sending ping request
[2012-08-01 14:44:29,393][TRACE][discovery.zen            ] [isolatednode] full ping responses: {none}
[2012-08-01 14:44:29,394][DEBUG][discovery.zen            ] [isolatednode] filtered ping responses: (filter_client[true], filter_data[false]) {none}
[2012-08-01 14:44:29,394][TRACE][discovery.zen.ping.multicast] [isolatednode] [16] sending ping request
[2012-08-01 14:44:30,895][TRACE][discovery.zen.ping.multicast] [isolatednode] [16] sending ping request
[2012-08-01 14:44:30,900][TRACE][discovery.zen.ping.multicast] [isolatednode] [16] received ping_response{target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]], cluster_name[splitbrain]}
[2012-08-01 14:44:31,911][TRACE][discovery.zen.ping.multicast] [isolatednode] [16] received ping_response{target [[node2][zzznode2][inet[/192.168.7.132:9301]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]], cluster_name[splitbrain]}
[2012-08-01 14:44:32,396][TRACE][discovery.zen            ] [isolatednode] full ping responses:
	--> target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]]
	--> target [[node2][zzznode2][inet[/192.168.7.132:9301]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]]
[2012-08-01 14:44:32,396][DEBUG][discovery.zen            ] [isolatednode] filtered ping responses: (filter_client[true], filter_data[false])
	--> target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]]
	--> target [[node2][zzznode2][inet[/192.168.7.132:9301]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]]
[2012-08-01 14:44:32,402][DEBUG][discovery.zen.fd         ] [isolatednode] [master] starting fault detection against master [[node1][zzznode1][inet[/192.168.7.132:9300]]], reason [initial_join]
[2012-08-01 14:44:32,404][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:44:32,404][DEBUG][cluster.service          ] [isolatednode] got first state from fresh master [zzznode1]
[2012-08-01 14:44:32,405][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [17], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], [R], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:44:32,405][INFO ][cluster.service          ] [isolatednode] detected_master [node1][zzznode1][inet[/192.168.7.132:9300]], added {[node2][zzznode2][inet[/192.168.7.132:9301]],[node1][zzznode1][inet[/192.168.7.132:9300]],}, reason: zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])
[2012-08-01 14:44:32,427][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:44:41,547][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:44:41,547][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [18], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING]
--------[test1][1], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
--------[test1][1], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:44:41,623][DEBUG][cluster.action.shard     ] [isolatednode] sending shard started for [test1][0], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:44:41,642][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:44:41,643][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:44:41,643][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [19], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[aaanode3], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], relocating [aaanode3], [P], s[RELOCATING]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], relocating [aaanode3], [P], s[RELOCATING]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [R], s[STARTED]
--------[test1][1], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING]
--------[test1][2], node[aaanode3], relocating [zzznode2], [P], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:44:41,652][DEBUG][cluster.action.shard     ] [isolatednode] sending shard started for [test1][1], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:44:41,675][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:44:41,676][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:44:41,677][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [20], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[aaanode3], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], relocating [aaanode3], [P], s[RELOCATING]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], relocating [aaanode3], [P], s[RELOCATING]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [R], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[aaanode3], relocating [zzznode2], [P], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:44:41,678][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:44:41,686][DEBUG][cluster.action.shard     ] [isolatednode] sending shard started for [test1][2], node[aaanode3], relocating [zzznode2], [P], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:44:41,691][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:44:41,691][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [21], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[aaanode3], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [R], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:44:41,693][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:45:13,464][TRACE][discovery.zen.fd         ] [isolatednode] [master] failed to ping [[node1][zzznode1][inet[/192.168.7.132:9300]]], retry [1] out of [2]
org.elasticsearch.transport.ReceiveTimeoutTransportException: [node1][inet[/192.168.7.132:9300]][discovery/zen/fd/masterPing] request_id [155] timed out after [3001ms]
	at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:347)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:45:16,466][TRACE][discovery.zen.fd         ] [isolatednode] [master] failed to ping [[node1][zzznode1][inet[/192.168.7.132:9300]]], retry [2] out of [2]
org.elasticsearch.transport.ReceiveTimeoutTransportException: [node1][inet[/192.168.7.132:9300]][discovery/zen/fd/masterPing] request_id [156] timed out after [3001ms]
	at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:347)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:45:16,467][DEBUG][discovery.zen.fd         ] [isolatednode] [master] failed to ping [[node1][zzznode1][inet[/192.168.7.132:9300]]], tried [2] times, each with maximum [3s] timeout
[2012-08-01 14:45:16,467][DEBUG][discovery.zen.fd         ] [isolatednode] [master] stopping fault detection against master [[node1][zzznode1][inet[/192.168.7.132:9300]]], reason [master failure, failed to ping, tried [2] times, each with  maximum [3s] timeout]
[2012-08-01 14:45:16,467][INFO ][discovery.zen            ] [isolatednode] master_left [[node1][zzznode1][inet[/192.168.7.132:9300]]], reason [failed to ping, tried [2] times, each with  maximum [3s] timeout]
[2012-08-01 14:45:16,467][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-master_failed ([node1][zzznode1][inet[/192.168.7.132:9300]])]: execute
[2012-08-01 14:45:16,468][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [22], source [zen-disco-master_failed ([node1][zzznode1][inet[/192.168.7.132:9300]])]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local, master
   [node2][zzznode2][inet[/192.168.7.132:9301]]
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[aaanode3], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [R], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode1][X]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:45:16,468][INFO ][cluster.service          ] [isolatednode] master {new [isolatednode][aaanode3][inet[/192.168.7.130:9300]], previous [node1][zzznode1][inet[/192.168.7.132:9300]]}, removed {[node1][zzznode1][inet[/192.168.7.132:9300]],}, reason: zen-disco-master_failed ([node1][zzznode1][inet[/192.168.7.132:9300]])
[2012-08-01 14:45:16,469][DEBUG][cluster.service          ] [isolatednode] processing [zen-disco-master_failed ([node1][zzznode1][inet[/192.168.7.132:9300]])]: done applying updated cluster_state
[2012-08-01 14:45:16,469][DEBUG][cluster.service          ] [isolatednode] processing [routing-table-updater]: execute
[2012-08-01 14:46:11,593][WARN ][transport.netty          ] [isolatednode] Exception caught on netty layer [[id: 0x20d75cf7, /192.168.7.130:52901 => /192.168.7.132:9301]]
java.io.IOException: Operation timed out
	at sun.nio.ch.FileDispatcher.read0(Native Method)
	at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:21)
	at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:198)
	at sun.nio.ch.IOUtil.read(IOUtil.java:166)
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:245)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:63)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:46:11,593][WARN ][transport.netty          ] [isolatednode] Exception caught on netty layer [[id: 0x4665d588, /192.168.7.130:52903 => /192.168.7.132:9301]]
java.io.IOException: Operation timed out
	at sun.nio.ch.FileDispatcher.read0(Native Method)
	at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:21)
	at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:198)
	at sun.nio.ch.IOUtil.read(IOUtil.java:166)
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:245)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:63)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:46:11,600][DEBUG][discovery.zen.publish    ] [isolatednode] failed to send cluster state to [[node2][zzznode2][inet[/192.168.7.132:9301]]], should be detected as failed soon...
org.elasticsearch.transport.NodeDisconnectedException: [node2][inet[/192.168.7.132:9301]][discovery/zen/publish] disconnected
[2012-08-01 14:46:11,602][TRACE][cluster.service          ] [isolatednode] cluster state updated:
version [23], source [routing-table-updater]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]], local, master
   [node2][zzznode2][inet[/192.168.7.132:9301]]
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[aaanode3], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [R], s[INITIALIZING]
--------[test1][2], node[aaanode3], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[aaanode3], [R], s[INITIALIZING]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [P], s[STARTED]
--------[test1][4], node[aaanode3], [R], s[INITIALIZING]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [R], s[INITIALIZING]
--------[test1][3], node[zzznode2], [P], s[STARTED]
--------[test1][4], node[zzznode2], [P], s[STARTED]
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [R], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[aaanode3], [P], s[STARTED]
--------[test1][3], node[aaanode3], [R], s[INITIALIZING]
--------[test1][4], node[aaanode3], [R], s[INITIALIZING]
---- unassigned

[2012-08-01 14:46:11,603][DEBUG][discovery.zen.publish    ] [isolatednode] failed to send cluster state to [[node2][zzznode2][inet[/192.168.7.132:9301]]], should be detected as failed soon...
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][discovery/zen/publish]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.discovery.zen.publish.PublishClusterStateAction.publish(PublishClusterStateAction.java:89)
	at org.elasticsearch.discovery.zen.ZenDiscovery.publish(ZenDiscovery.java:268)
	at org.elasticsearch.discovery.DiscoveryService.publish(DiscoveryService.java:115)
	at org.elasticsearch.cluster.service.InternalClusterService$2.run(InternalClusterService.java:293)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 7 more
[2012-08-01 14:46:11,667][DEBUG][cluster.service          ] [isolatednode] processing [routing-table-updater]: done applying updated cluster_state
[2012-08-01 14:46:17,166][WARN ][cluster.service          ] [isolatednode] failed to reconnect to node [node2][zzznode2][inet[/192.168.7.132:9301]]
org.elasticsearch.transport.ConnectTransportException: [node2][inet[/192.168.7.132:9301]] connect_timeout[2s]
	at org.elasticsearch.transport.netty.NettyTransport.connectToChannels(NettyTransport.java:594)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:536)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:514)
	at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:128)
	at org.elasticsearch.cluster.service.InternalClusterService$ReconnectToNodes.run(InternalClusterService.java:380)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: java.net.ConnectException: connection timed out
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.processConnectTimeout(NioClientSocketPipelineSink.java:391)
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.run(NioClientSocketPipelineSink.java:289)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	... 3 more
[2012-08-01 14:46:21,791][DEBUG][action.admin.indices.status] [isolatednode] [test1][0], node[zzznode2], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@5f5660ef]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:46:21,802][DEBUG][action.admin.cluster.node.info] [isolatednode] failed to execute on node [zzznode2]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][cluster/nodes/info/n]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.start(TransportNodesOperationAction.java:172)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.access$300(TransportNodesOperationAction.java:102)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:73)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:43)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeClusterAdminClient.execute(NodeClusterAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractClusterAdminClient.nodesInfo(AbstractClusterAdminClient.java:135)
	at org.elasticsearch.rest.action.admin.cluster.node.info.RestNodesInfoAction.executeNodeRequest(RestNodesInfoAction.java:106)
	at org.elasticsearch.rest.action.admin.cluster.node.info.RestNodesInfoAction.handleRequest(RestNodesInfoAction.java:101)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 43 more
[2012-08-01 14:46:21,801][DEBUG][action.admin.cluster.node.stats] [isolatednode] failed to execute on node [zzznode2]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][cluster/nodes/stats/n]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.start(TransportNodesOperationAction.java:172)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.access$300(TransportNodesOperationAction.java:102)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:73)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:43)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeClusterAdminClient.execute(NodeClusterAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractClusterAdminClient.nodesStats(AbstractClusterAdminClient.java:150)
	at org.elasticsearch.rest.action.admin.cluster.node.stats.RestNodesStatsAction.executeNodeStats(RestNodesStatsAction.java:130)
	at org.elasticsearch.rest.action.admin.cluster.node.stats.RestNodesStatsAction.handleRequest(RestNodesStatsAction.java:125)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 43 more
[2012-08-01 14:46:21,799][DEBUG][action.admin.indices.status] [isolatednode] [test1][4], node[zzznode2], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@5f5660ef]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:46:21,799][DEBUG][action.admin.indices.status] [isolatednode] [test1][3], node[aaanode3], [R], s[INITIALIZING]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@5f5660ef]
org.elasticsearch.index.IndexShardMissingException: [test1][3] missing
	at org.elasticsearch.index.service.InternalIndexService.shardSafe(InternalIndexService.java:179)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:153)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:59)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:234)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction$1.run(TransportBroadcastOperationAction.java:187)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:46:21,794][DEBUG][action.admin.indices.status] [isolatednode] [test1][3], node[zzznode2], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@5f5660ef]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:46:21,791][DEBUG][action.admin.indices.status] [isolatednode] [test1][2], node[zzznode2], [R], s[INITIALIZING]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@5f5660ef]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:46:21,791][DEBUG][action.admin.indices.status] [isolatednode] [test1][1], node[zzznode2], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@5f5660ef]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:46:21,810][DEBUG][action.admin.indices.status] [isolatednode] [test1][4], node[aaanode3], [R], s[INITIALIZING]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@5f5660ef]
org.elasticsearch.index.IndexShardMissingException: [test1][4] missing
	at org.elasticsearch.index.service.InternalIndexService.shardSafe(InternalIndexService.java:179)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:153)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:59)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:234)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction$1.run(TransportBroadcastOperationAction.java:187)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:46:22,096][DEBUG][action.admin.indices.status] [isolatednode] [test1][0], node[zzznode2], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@533c2ac3]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:46:22,100][DEBUG][action.admin.indices.status] [isolatednode] [test1][3], node[aaanode3], [R], s[INITIALIZING]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@533c2ac3]
org.elasticsearch.index.IndexShardMissingException: [test1][3] missing
	at org.elasticsearch.index.service.InternalIndexService.shardSafe(InternalIndexService.java:179)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:153)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:59)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:234)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction$1.run(TransportBroadcastOperationAction.java:187)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:46:22,102][DEBUG][action.admin.indices.status] [isolatednode] [test1][4], node[aaanode3], [R], s[INITIALIZING]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@533c2ac3]
org.elasticsearch.index.IndexShardMissingException: [test1][4] missing
	at org.elasticsearch.index.service.InternalIndexService.shardSafe(InternalIndexService.java:179)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:153)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:59)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:234)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction$1.run(TransportBroadcastOperationAction.java:187)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:46:22,099][DEBUG][action.admin.cluster.node.stats] [isolatednode] failed to execute on node [zzznode2]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][cluster/nodes/stats/n]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.start(TransportNodesOperationAction.java:172)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.access$300(TransportNodesOperationAction.java:102)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:73)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:43)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeClusterAdminClient.execute(NodeClusterAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractClusterAdminClient.nodesStats(AbstractClusterAdminClient.java:150)
	at org.elasticsearch.rest.action.admin.cluster.node.stats.RestNodesStatsAction.executeNodeStats(RestNodesStatsAction.java:130)
	at org.elasticsearch.rest.action.admin.cluster.node.stats.RestNodesStatsAction.handleRequest(RestNodesStatsAction.java:125)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 43 more
[2012-08-01 14:46:22,099][DEBUG][action.admin.indices.status] [isolatednode] [test1][1], node[zzznode2], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@533c2ac3]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:46:22,099][DEBUG][action.admin.indices.status] [isolatednode] [test1][4], node[zzznode2], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@533c2ac3]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:46:22,097][DEBUG][action.admin.indices.status] [isolatednode] [test1][3], node[zzznode2], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@533c2ac3]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:46:22,097][DEBUG][action.admin.indices.status] [isolatednode] [test1][2], node[zzznode2], [R], s[INITIALIZING]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@533c2ac3]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:46:22,097][DEBUG][action.admin.cluster.node.info] [isolatednode] failed to execute on node [zzznode2]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][cluster/nodes/info/n]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.start(TransportNodesOperationAction.java:172)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.access$300(TransportNodesOperationAction.java:102)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:73)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:43)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeClusterAdminClient.execute(NodeClusterAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractClusterAdminClient.nodesInfo(AbstractClusterAdminClient.java:135)
	at org.elasticsearch.rest.action.admin.cluster.node.info.RestNodesInfoAction.executeNodeRequest(RestNodesInfoAction.java:106)
	at org.elasticsearch.rest.action.admin.cluster.node.info.RestNodesInfoAction.handleRequest(RestNodesInfoAction.java:101)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 43 more
[2012-08-01 14:46:29,170][WARN ][cluster.service          ] [isolatednode] failed to reconnect to node [node2][zzznode2][inet[/192.168.7.132:9301]]
org.elasticsearch.transport.ConnectTransportException: [node2][inet[/192.168.7.132:9301]] connect_timeout[2s]
	at org.elasticsearch.transport.netty.NettyTransport.connectToChannels(NettyTransport.java:594)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:536)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:514)
	at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:128)
	at org.elasticsearch.cluster.service.InternalClusterService$ReconnectToNodes.run(InternalClusterService.java:380)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: java.net.ConnectException: connection timed out
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.processConnectTimeout(NioClientSocketPipelineSink.java:391)
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.run(NioClientSocketPipelineSink.java:289)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	... 3 more
[2012-08-01 14:46:41,184][WARN ][cluster.service          ] [isolatednode] failed to reconnect to node [node2][zzznode2][inet[/192.168.7.132:9301]]
org.elasticsearch.transport.ConnectTransportException: [node2][inet[/192.168.7.132:9301]] connect_timeout[2s]
	at org.elasticsearch.transport.netty.NettyTransport.connectToChannels(NettyTransport.java:594)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:536)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:514)
	at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:128)
	at org.elasticsearch.cluster.service.InternalClusterService$ReconnectToNodes.run(InternalClusterService.java:380)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: java.net.ConnectException: connection timed out
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.processConnectTimeout(NioClientSocketPipelineSink.java:391)
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.run(NioClientSocketPipelineSink.java:289)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	... 3 more
[2012-08-01 14:46:53,192][WARN ][cluster.service          ] [isolatednode] failed to reconnect to node [node2][zzznode2][inet[/192.168.7.132:9301]]
org.elasticsearch.transport.ConnectTransportException: [node2][inet[/192.168.7.132:9301]] connect_timeout[2s]
	at org.elasticsearch.transport.netty.NettyTransport.connectToChannels(NettyTransport.java:594)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:536)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:514)
	at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:128)
	at org.elasticsearch.cluster.service.InternalClusterService$ReconnectToNodes.run(InternalClusterService.java:380)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: java.net.ConnectException: connection timed out
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.processConnectTimeout(NioClientSocketPipelineSink.java:391)
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.run(NioClientSocketPipelineSink.java:289)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	... 3 more
[2012-08-01 14:47:05,201][WARN ][cluster.service          ] [isolatednode] failed to reconnect to node [node2][zzznode2][inet[/192.168.7.132:9301]]
org.elasticsearch.transport.ConnectTransportException: [node2][inet[/192.168.7.132:9301]] connect_timeout[2s]
	at org.elasticsearch.transport.netty.NettyTransport.connectToChannels(NettyTransport.java:594)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:536)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:514)
	at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:128)
	at org.elasticsearch.cluster.service.InternalClusterService$ReconnectToNodes.run(InternalClusterService.java:380)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: java.net.ConnectException: connection timed out
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.processConnectTimeout(NioClientSocketPipelineSink.java:391)
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.run(NioClientSocketPipelineSink.java:289)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	... 3 more
[2012-08-01 14:47:17,208][WARN ][cluster.service          ] [isolatednode] failed to reconnect to node [node2][zzznode2][inet[/192.168.7.132:9301]]
org.elasticsearch.transport.ConnectTransportException: [node2][inet[/192.168.7.132:9301]] connect_timeout[2s]
	at org.elasticsearch.transport.netty.NettyTransport.connectToChannels(NettyTransport.java:594)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:536)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:514)
	at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:128)
	at org.elasticsearch.cluster.service.InternalClusterService$ReconnectToNodes.run(InternalClusterService.java:380)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: java.net.ConnectException: connection timed out
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.processConnectTimeout(NioClientSocketPipelineSink.java:391)
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.run(NioClientSocketPipelineSink.java:289)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	... 3 more
[2012-08-01 14:47:29,219][WARN ][cluster.service          ] [isolatednode] failed to reconnect to node [node2][zzznode2][inet[/192.168.7.132:9301]]
org.elasticsearch.transport.ConnectTransportException: [node2][inet[/192.168.7.132:9301]] connect_timeout[2s]
	at org.elasticsearch.transport.netty.NettyTransport.connectToChannels(NettyTransport.java:594)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:536)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:514)
	at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:128)
	at org.elasticsearch.cluster.service.InternalClusterService$ReconnectToNodes.run(InternalClusterService.java:380)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: java.net.ConnectException: connection timed out
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.processConnectTimeout(NioClientSocketPipelineSink.java:391)
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.run(NioClientSocketPipelineSink.java:289)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	... 3 more
[2012-08-01 14:47:41,233][WARN ][cluster.service          ] [isolatednode] failed to reconnect to node [node2][zzznode2][inet[/192.168.7.132:9301]]
org.elasticsearch.transport.ConnectTransportException: [node2][inet[/192.168.7.132:9301]] connect_timeout[2s]
	at org.elasticsearch.transport.netty.NettyTransport.connectToChannels(NettyTransport.java:594)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:536)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:514)
	at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:128)
	at org.elasticsearch.cluster.service.InternalClusterService$ReconnectToNodes.run(InternalClusterService.java:380)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: java.net.ConnectException: connection timed out
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.processConnectTimeout(NioClientSocketPipelineSink.java:391)
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.run(NioClientSocketPipelineSink.java:289)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	... 3 more
[2012-08-01 14:47:53,239][WARN ][cluster.service          ] [isolatednode] failed to reconnect to node [node2][zzznode2][inet[/192.168.7.132:9301]]
org.elasticsearch.transport.ConnectTransportException: [node2][inet[/192.168.7.132:9301]] connect_timeout[2s]
	at org.elasticsearch.transport.netty.NettyTransport.connectToChannels(NettyTransport.java:594)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:536)
	at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:514)
	at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:128)
	at org.elasticsearch.cluster.service.InternalClusterService$ReconnectToNodes.run(InternalClusterService.java:380)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: java.net.ConnectException: connection timed out
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.processConnectTimeout(NioClientSocketPipelineSink.java:391)
	at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.run(NioClientSocketPipelineSink.java:289)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	... 3 more
[2012-08-01 14:47:55,274][DEBUG][action.admin.indices.status] [isolatednode] [test1][0], node[zzznode2], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@be7f971]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:47:55,279][DEBUG][action.admin.cluster.node.stats] [isolatednode] failed to execute on node [zzznode2]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][cluster/nodes/stats/n]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.start(TransportNodesOperationAction.java:172)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.access$300(TransportNodesOperationAction.java:102)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:73)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:43)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeClusterAdminClient.execute(NodeClusterAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractClusterAdminClient.nodesStats(AbstractClusterAdminClient.java:150)
	at org.elasticsearch.rest.action.admin.cluster.node.stats.RestNodesStatsAction.executeNodeStats(RestNodesStatsAction.java:130)
	at org.elasticsearch.rest.action.admin.cluster.node.stats.RestNodesStatsAction.handleRequest(RestNodesStatsAction.java:125)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 43 more
[2012-08-01 14:47:55,274][DEBUG][action.admin.indices.status] [isolatednode] [test1][3], node[zzznode2], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@be7f971]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:47:55,278][DEBUG][action.admin.indices.status] [isolatednode] [test1][3], node[aaanode3], [R], s[INITIALIZING]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@be7f971]
org.elasticsearch.index.IndexShardMissingException: [test1][3] missing
	at org.elasticsearch.index.service.InternalIndexService.shardSafe(InternalIndexService.java:179)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:153)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:59)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:234)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction$1.run(TransportBroadcastOperationAction.java:187)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:47:55,298][DEBUG][action.admin.indices.status] [isolatednode] [test1][4], node[aaanode3], [R], s[INITIALIZING]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@be7f971]
org.elasticsearch.index.IndexShardMissingException: [test1][4] missing
	at org.elasticsearch.index.service.InternalIndexService.shardSafe(InternalIndexService.java:179)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:153)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:59)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:234)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction$1.run(TransportBroadcastOperationAction.java:187)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:47:55,278][DEBUG][action.admin.cluster.node.info] [isolatednode] failed to execute on node [zzznode2]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][cluster/nodes/info/n]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.start(TransportNodesOperationAction.java:172)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.access$300(TransportNodesOperationAction.java:102)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:73)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:43)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeClusterAdminClient.execute(NodeClusterAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractClusterAdminClient.nodesInfo(AbstractClusterAdminClient.java:135)
	at org.elasticsearch.rest.action.admin.cluster.node.info.RestNodesInfoAction.executeNodeRequest(RestNodesInfoAction.java:106)
	at org.elasticsearch.rest.action.admin.cluster.node.info.RestNodesInfoAction.handleRequest(RestNodesInfoAction.java:101)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 43 more
[2012-08-01 14:47:55,278][DEBUG][action.admin.indices.status] [isolatednode] [test1][4], node[zzznode2], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@be7f971]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:47:55,276][DEBUG][action.admin.indices.status] [isolatednode] [test1][2], node[zzznode2], [R], s[INITIALIZING]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@be7f971]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:47:55,276][DEBUG][action.admin.indices.status] [isolatednode] [test1][1], node[zzznode2], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@be7f971]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:47:55,900][DEBUG][action.admin.cluster.node.info] [isolatednode] failed to execute on node [zzznode2]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][cluster/nodes/info/n]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.start(TransportNodesOperationAction.java:172)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.access$300(TransportNodesOperationAction.java:102)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:73)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:43)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeClusterAdminClient.execute(NodeClusterAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractClusterAdminClient.nodesInfo(AbstractClusterAdminClient.java:135)
	at org.elasticsearch.rest.action.admin.cluster.node.info.RestNodesInfoAction.executeNodeRequest(RestNodesInfoAction.java:106)
	at org.elasticsearch.rest.action.admin.cluster.node.info.RestNodesInfoAction.handleRequest(RestNodesInfoAction.java:101)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 43 more
[2012-08-01 14:47:55,904][DEBUG][action.admin.cluster.node.stats] [isolatednode] failed to execute on node [zzznode2]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][cluster/nodes/stats/n]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.start(TransportNodesOperationAction.java:172)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.access$300(TransportNodesOperationAction.java:102)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:73)
	at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:43)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeClusterAdminClient.execute(NodeClusterAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractClusterAdminClient.nodesStats(AbstractClusterAdminClient.java:150)
	at org.elasticsearch.rest.action.admin.cluster.node.stats.RestNodesStatsAction.executeNodeStats(RestNodesStatsAction.java:130)
	at org.elasticsearch.rest.action.admin.cluster.node.stats.RestNodesStatsAction.handleRequest(RestNodesStatsAction.java:125)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 43 more
[2012-08-01 14:47:55,900][DEBUG][action.admin.indices.status] [isolatednode] [test1][3], node[zzznode2], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@53c7378f]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:47:55,900][DEBUG][action.admin.indices.status] [isolatednode] [test1][3], node[aaanode3], [R], s[INITIALIZING]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@53c7378f]
org.elasticsearch.index.IndexShardMissingException: [test1][3] missing
	at org.elasticsearch.index.service.InternalIndexService.shardSafe(InternalIndexService.java:179)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:153)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:59)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:234)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction$1.run(TransportBroadcastOperationAction.java:187)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:47:55,913][DEBUG][action.admin.indices.status] [isolatednode] [test1][4], node[aaanode3], [R], s[INITIALIZING]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@53c7378f]
org.elasticsearch.index.IndexShardMissingException: [test1][4] missing
	at org.elasticsearch.index.service.InternalIndexService.shardSafe(InternalIndexService.java:179)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:153)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:59)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:234)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction$1.run(TransportBroadcastOperationAction.java:187)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:47:55,900][DEBUG][action.admin.indices.status] [isolatednode] [test1][2], node[zzznode2], [R], s[INITIALIZING]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@53c7378f]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:47:55,900][DEBUG][action.admin.indices.status] [isolatednode] [test1][1], node[zzznode2], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@53c7378f]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:47:55,900][DEBUG][action.admin.indices.status] [isolatednode] [test1][4], node[zzznode2], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@53c7378f]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:47:55,900][DEBUG][action.admin.indices.status] [isolatednode] [test1][0], node[zzznode2], [P], s[STARTED]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@53c7378f]
org.elasticsearch.transport.SendRequestTransportException: [node2][inet[/192.168.7.132:9301]][indices/status/s]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:200)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:172)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:245)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.start(TransportBroadcastOperationAction.java:169)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:73)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction.doExecute(TransportBroadcastOperationAction.java:44)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:61)
	at org.elasticsearch.client.node.NodeIndicesAdminClient.execute(NodeIndicesAdminClient.java:70)
	at org.elasticsearch.client.support.AbstractIndicesAdminClient.status(AbstractIndicesAdminClient.java:346)
	at org.elasticsearch.rest.action.admin.indices.status.RestIndicesStatusAction.handleRequest(RestIndicesStatusAction.java:70)
	at org.elasticsearch.rest.RestController.executeHandler(RestController.java:159)
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:142)
	at org.elasticsearch.http.HttpServer.internalDispatchRequest(HttpServer.java:120)
	at org.elasticsearch.http.HttpServer$Dispatcher.dispatchRequest(HttpServer.java:82)
	at org.elasticsearch.http.netty.NettyHttpServerTransport.dispatchRequest(NettyHttpServerTransport.java:256)
	at org.elasticsearch.http.netty.HttpRequestHandler.messageReceived(HttpRequestHandler.java:43)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:149)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
	at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:455)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:538)
	at org.elasticsearch.common.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:437)
	at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:75)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
	at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:563)
	at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:558)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
	at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:91)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.processSelectedKeys(AbstractNioWorker.java:373)
	at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:247)
	at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:35)
	at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
	at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Caused by: org.elasticsearch.transport.NodeNotConnectedException: [node2][inet[/192.168.7.132:9301]] Node not connected
	at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:673)
	at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:476)
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:185)
	... 44 more
[2012-08-01 14:48:09,167][DEBUG][action.admin.indices.status] [isolatednode] [test1][3], node[aaanode3], [R], s[INITIALIZING]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@6a07348e]
org.elasticsearch.index.IndexShardMissingException: [test1][3] missing
	at org.elasticsearch.index.service.InternalIndexService.shardSafe(InternalIndexService.java:179)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:153)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:59)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:234)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction$1.run(TransportBroadcastOperationAction.java:187)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:48:09,170][DEBUG][action.admin.indices.status] [isolatednode] [test1][4], node[aaanode3], [R], s[INITIALIZING]: Failed to execute [org.elasticsearch.action.admin.indices.status.IndicesStatusRequest@6a07348e]
org.elasticsearch.index.IndexShardMissingException: [test1][4] missing
	at org.elasticsearch.index.service.InternalIndexService.shardSafe(InternalIndexService.java:179)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:153)
	at org.elasticsearch.action.admin.indices.status.TransportIndicesStatusAction.shardOperation(TransportIndicesStatusAction.java:59)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:234)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction.performOperation(TransportBroadcastOperationAction.java:211)
	at org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction$AsyncBroadcastAction$1.run(TransportBroadcastOperationAction.java:187)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
[2012-08-01 14:48:39,541][INFO ][node                     ] [isolatednode] {0.20.0.Beta1-SNAPSHOT}[3236]: stopping ...
[2012-08-01 14:48:39,574][INFO ][node                     ] [isolatednode] {0.20.0.Beta1-SNAPSHOT}[3236]: stopped
[2012-08-01 14:48:39,575][INFO ][node                     ] [isolatednode] {0.20.0.Beta1-SNAPSHOT}[3236]: closing ...
[2012-08-01 14:48:39,584][INFO ][node                     ] [isolatednode] {0.20.0.Beta1-SNAPSHOT}[3236]: closed

splitbrain-node1.log

[2012-08-01 14:40:02,305][INFO ][node                     ] [node1] {0.20.0.Beta1-SNAPSHOT}[32215]: initializing ...
[2012-08-01 14:40:02,310][INFO ][plugins                  ] [node1] loaded [], sites []
[2012-08-01 14:40:03,377][DEBUG][discovery.zen.ping.multicast] [node1] using group [224.2.2.4], with port [54328], ttl [3], and address [null]
[2012-08-01 14:40:03,380][DEBUG][discovery.zen.ping.unicast] [node1] using initial hosts [], with concurrent_connects [10]
[2012-08-01 14:40:03,381][DEBUG][discovery.zen            ] [node1] using ping.timeout [3s], master_election.filter_client [true], master_election.filter_data [false]
[2012-08-01 14:40:03,385][DEBUG][discovery.zen.elect      ] [node1] using minimum_master_nodes [2]
[2012-08-01 14:40:03,386][DEBUG][discovery.zen.fd         ] [node1] [master] uses ping_interval [1s], ping_timeout [3s], ping_retries [2]
[2012-08-01 14:40:03,390][DEBUG][discovery.zen.fd         ] [node1] [node  ] uses ping_interval [1s], ping_timeout [3s], ping_retries [2]
[2012-08-01 14:40:04,155][DEBUG][cluster.routing.allocation.decider] [node1] using node_concurrent_recoveries [2], node_initial_primaries_recoveries [4]
[2012-08-01 14:40:04,155][DEBUG][cluster.routing.allocation.decider] [node1] using [cluster.routing.allocation.allow_rebalance] with [indices_all_active]
[2012-08-01 14:40:04,156][DEBUG][cluster.routing.allocation.decider] [node1] using [cluster_concurrent_rebalance] with [2]
[2012-08-01 14:40:04,322][INFO ][node                     ] [node1] {0.20.0.Beta1-SNAPSHOT}[32215]: initialized
[2012-08-01 14:40:04,323][INFO ][node                     ] [node1] {0.20.0.Beta1-SNAPSHOT}[32215]: starting ...
[2012-08-01 14:40:04,386][INFO ][transport                ] [node1] bound_address {inet[/0:0:0:0:0:0:0:0:9300]}, publish_address {inet[/192.168.7.132:9300]}
[2012-08-01 14:40:04,397][TRACE][discovery                ] [node1] waiting for 30s for the initial state to be set by the discovery
[2012-08-01 14:40:04,408][TRACE][discovery.zen.ping.multicast] [node1] [1] sending ping request
[2012-08-01 14:40:05,910][TRACE][discovery.zen.ping.multicast] [node1] [1] sending ping request
[2012-08-01 14:40:07,414][TRACE][discovery.zen            ] [node1] full ping responses: {none}
[2012-08-01 14:40:07,414][DEBUG][discovery.zen            ] [node1] filtered ping responses: (filter_client[true], filter_data[false]) {none}
[2012-08-01 14:40:07,414][TRACE][discovery.zen.ping.multicast] [node1] [2] sending ping request
[2012-08-01 14:40:08,506][TRACE][discovery.zen.ping.multicast] [node1] [1] received ping_request from [[node2][zzznode2][inet[/192.168.7.132:9301]]], sending ping_response{target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [null], cluster_name[splitbrain]}
[2012-08-01 14:40:08,915][TRACE][discovery.zen.ping.multicast] [node1] [2] sending ping request
[2012-08-01 14:40:08,926][TRACE][discovery.zen.ping.multicast] [node1] [2] received ping_response{target [[node2][zzznode2][inet[/192.168.7.132:9301]]], master [null], cluster_name[splitbrain]}
[2012-08-01 14:40:10,007][TRACE][discovery.zen.ping.multicast] [node1] [1] received ping_request from [[node2][zzznode2][inet[/192.168.7.132:9301]]], sending ping_response{target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [null], cluster_name[splitbrain]}
[2012-08-01 14:40:10,415][TRACE][discovery.zen            ] [node1] full ping responses:
	--> target [[node2][zzznode2][inet[/192.168.7.132:9301]]], master [null]
[2012-08-01 14:40:10,415][DEBUG][discovery.zen            ] [node1] filtered ping responses: (filter_client[true], filter_data[false])
	--> target [[node2][zzznode2][inet[/192.168.7.132:9301]]], master [null]
[2012-08-01 14:40:10,418][DEBUG][cluster.service          ] [node1] processing [zen-disco-join (elected_as_master)]: execute
[2012-08-01 14:40:10,418][TRACE][cluster.service          ] [node1] cluster state updated:
version [1], source [zen-disco-join (elected_as_master)]
nodes: 
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
routing_nodes:
---- unassigned

[2012-08-01 14:40:10,419][INFO ][cluster.service          ] [node1] new_master [node1][zzznode1][inet[/192.168.7.132:9300]], reason: zen-disco-join (elected_as_master)
[2012-08-01 14:40:10,430][DEBUG][cluster.service          ] [node1] processing [zen-disco-join (elected_as_master)]: done applying updated cluster_state
[2012-08-01 14:40:10,430][TRACE][discovery                ] [node1] initial state set from discovery
[2012-08-01 14:40:10,431][INFO ][discovery                ] [node1] splitbrain/zzznode1
[2012-08-01 14:40:10,434][INFO ][http                     ] [node1] bound_address {inet[/0:0:0:0:0:0:0:0:9200]}, publish_address {inet[/192.168.7.132:9200]}
[2012-08-01 14:40:10,434][INFO ][node                     ] [node1] {0.20.0.Beta1-SNAPSHOT}[32215]: started
[2012-08-01 14:40:11,512][TRACE][discovery.zen.ping.multicast] [node1] [2] received ping_request from [[node2][zzznode2][inet[/192.168.7.132:9301]]], sending ping_response{target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]], cluster_name[splitbrain]}
[2012-08-01 14:40:13,012][TRACE][discovery.zen.ping.multicast] [node1] [2] received ping_request from [[node2][zzznode2][inet[/192.168.7.132:9301]]], sending ping_response{target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]], cluster_name[splitbrain]}
[2012-08-01 14:40:14,522][DEBUG][cluster.service          ] [node1] processing [zen-disco-receive(join from node[[node2][zzznode2][inet[/192.168.7.132:9301]]])]: execute
[2012-08-01 14:40:14,524][TRACE][cluster.service          ] [node1] cluster state updated:
version [2], source [zen-disco-receive(join from node[[node2][zzznode2][inet[/192.168.7.132:9301]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
routing_nodes:
---- unassigned

[2012-08-01 14:40:14,524][INFO ][cluster.service          ] [node1] added {[node2][zzznode2][inet[/192.168.7.132:9301]],}, reason: zen-disco-receive(join from node[[node2][zzznode2][inet[/192.168.7.132:9301]]])
[2012-08-01 14:40:14,529][DEBUG][cluster.service          ] [node1] processing [zen-disco-receive(join from node[[node2][zzznode2][inet[/192.168.7.132:9301]]])]: done applying updated cluster_state
[2012-08-01 14:40:14,536][DEBUG][cluster.service          ] [node1] processing [local-gateway-elected-state]: execute
[2012-08-01 14:40:14,538][TRACE][cluster.service          ] [node1] cluster state updated:
version [3], source [local-gateway-elected-state]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
routing_nodes:
---- unassigned

[2012-08-01 14:40:14,609][INFO ][gateway                  ] [node1] recovered [0] indices into cluster_state
[2012-08-01 14:40:14,609][DEBUG][cluster.service          ] [node1] processing [local-gateway-elected-state]: done applying updated cluster_state
[2012-08-01 14:40:20,429][DEBUG][cluster.service          ] [node1] processing [routing-table-updater]: execute
[2012-08-01 14:40:20,430][DEBUG][cluster.service          ] [node1] processing [routing-table-updater]: no change in cluster_state
[2012-08-01 14:42:14,139][TRACE][discovery.zen.ping.multicast] [node1] [1] received ping_request from [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]], sending ping_response{target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]], cluster_name[splitbrain]}
[2012-08-01 14:42:15,642][TRACE][discovery.zen.ping.multicast] [node1] [1] received ping_request from [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]], sending ping_response{target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]], cluster_name[splitbrain]}
[2012-08-01 14:42:17,175][DEBUG][cluster.service          ] [node1] processing [zen-disco-receive(join from node[[isolatednode][aaanode3][inet[/192.168.7.130:9300]]])]: execute
[2012-08-01 14:42:17,176][TRACE][cluster.service          ] [node1] cluster state updated:
version [4], source [zen-disco-receive(join from node[[isolatednode][aaanode3][inet[/192.168.7.130:9300]]])]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
routing_nodes:
---- unassigned

[2012-08-01 14:42:17,176][INFO ][cluster.service          ] [node1] added {[isolatednode][aaanode3][inet[/192.168.7.130:9300]],}, reason: zen-disco-receive(join from node[[isolatednode][aaanode3][inet[/192.168.7.130:9300]]])
[2012-08-01 14:42:17,176][DEBUG][cluster.service          ] [node1] processing [zen-disco-receive(join from node[[isolatednode][aaanode3][inet[/192.168.7.130:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:20,431][DEBUG][cluster.service          ] [node1] processing [routing-table-updater]: execute
[2012-08-01 14:42:20,432][DEBUG][cluster.service          ] [node1] processing [routing-table-updater]: no change in cluster_state
[2012-08-01 14:42:34,753][DEBUG][cluster.service          ] [node1] processing [create-index [test1], cause [api]]: execute
[2012-08-01 14:42:35,015][INFO ][cluster.metadata         ] [node1] [test1] creating index, cause [api], shards [5]/[1], mappings []
[2012-08-01 14:42:35,029][TRACE][cluster.service          ] [node1] cluster state updated:
version [5], source [create-index [test1], cause [api]]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][0], node[null], [R], s[UNASSIGNED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[INITIALIZING]
--------[test1][1], node[null], [R], s[UNASSIGNED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
--------[test1][2], node[null], [R], s[UNASSIGNED]
----shard_id [test1][3]
--------[test1][3], node[zzznode1], [P], s[INITIALIZING]
--------[test1][3], node[null], [R], s[UNASSIGNED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[null], [R], s[UNASSIGNED]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
-----node_id[zzznode2][V]
--------[test1][1], node[zzznode2], [P], s[INITIALIZING]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][3], node[zzznode1], [P], s[INITIALIZING]
---- unassigned
--------[test1][0], node[null], [R], s[UNASSIGNED]
--------[test1][1], node[null], [R], s[UNASSIGNED]
--------[test1][2], node[null], [R], s[UNASSIGNED]
--------[test1][3], node[null], [R], s[UNASSIGNED]
--------[test1][4], node[null], [R], s[UNASSIGNED]

[2012-08-01 14:42:35,147][DEBUG][cluster.service          ] [node1] processing [create-index [test1], cause [api]]: done applying updated cluster_state
[2012-08-01 14:42:35,231][DEBUG][cluster.action.shard     ] [node1] sending shard started for [test1][3], node[zzznode1], [P], s[INITIALIZING], reason [after recovery from gateway]
[2012-08-01 14:42:35,232][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][3], node[zzznode1], [P], s[INITIALIZING], reason [after recovery from gateway]
[2012-08-01 14:42:35,233][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][3], node[zzznode1], [P], s[INITIALIZING]), reason [after recovery from gateway]]: execute
[2012-08-01 14:42:35,233][DEBUG][cluster.action.shard     ] [node1] applying started shards [[test1][3], node[zzznode1], [P], s[INITIALIZING]], reason [after recovery from gateway]
[2012-08-01 14:42:35,234][TRACE][cluster.service          ] [node1] cluster state updated:
version [6], source [shard-started ([test1][3], node[zzznode1], [P], s[INITIALIZING]), reason [after recovery from gateway]]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][0], node[null], [R], s[UNASSIGNED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[INITIALIZING]
--------[test1][1], node[null], [R], s[UNASSIGNED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
--------[test1][2], node[null], [R], s[UNASSIGNED]
----shard_id [test1][3]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][3], node[null], [R], s[UNASSIGNED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[null], [R], s[UNASSIGNED]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
-----node_id[zzznode2][V]
--------[test1][1], node[zzznode2], [P], s[INITIALIZING]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][3], node[zzznode1], [P], s[STARTED]
---- unassigned
--------[test1][0], node[null], [R], s[UNASSIGNED]
--------[test1][1], node[null], [R], s[UNASSIGNED]
--------[test1][2], node[null], [R], s[UNASSIGNED]
--------[test1][3], node[null], [R], s[UNASSIGNED]
--------[test1][4], node[null], [R], s[UNASSIGNED]

[2012-08-01 14:42:35,247][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][3], node[zzznode1], [P], s[INITIALIZING]), reason [after recovery from gateway]]: done applying updated cluster_state
[2012-08-01 14:42:35,394][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][1], node[zzznode2], [P], s[INITIALIZING], reason [after recovery from gateway]
[2012-08-01 14:42:35,395][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][1], node[zzznode2], [P], s[INITIALIZING]), reason [after recovery from gateway]]: execute
[2012-08-01 14:42:35,395][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][0], node[aaanode3], [P], s[INITIALIZING], reason [after recovery from gateway]
[2012-08-01 14:42:35,395][DEBUG][cluster.action.shard     ] [node1] applying started shards [[test1][1], node[zzznode2], [P], s[INITIALIZING]], reason [after recovery from gateway]
[2012-08-01 14:42:35,396][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][4], node[aaanode3], [P], s[INITIALIZING], reason [after recovery from gateway]
[2012-08-01 14:42:35,398][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][0], node[aaanode3], [P], s[INITIALIZING], reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]
[2012-08-01 14:42:35,398][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][4], node[aaanode3], [P], s[INITIALIZING], reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]
[2012-08-01 14:42:35,399][TRACE][cluster.service          ] [node1] cluster state updated:
version [7], source [shard-started ([test1][1], node[zzznode2], [P], s[INITIALIZING]), reason [after recovery from gateway]]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][0], node[null], [R], s[UNASSIGNED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
--------[test1][2], node[null], [R], s[UNASSIGNED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[INITIALIZING]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[null], [R], s[UNASSIGNED]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
-----node_id[zzznode2][V]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
--------[test1][3], node[zzznode2], [R], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
--------[test1][3], node[zzznode1], [P], s[STARTED]
---- unassigned
--------[test1][0], node[null], [R], s[UNASSIGNED]
--------[test1][2], node[null], [R], s[UNASSIGNED]
--------[test1][4], node[null], [R], s[UNASSIGNED]

[2012-08-01 14:42:35,405][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][0], node[aaanode3], [P], s[INITIALIZING], reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]
[2012-08-01 14:42:35,405][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][4], node[aaanode3], [P], s[INITIALIZING], reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]
[2012-08-01 14:42:35,405][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][1], node[zzznode2], [P], s[INITIALIZING], reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]
[2012-08-01 14:42:35,413][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][2], node[zzznode2], [P], s[INITIALIZING], reason [after recovery from gateway]
[2012-08-01 14:42:35,445][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][1], node[zzznode2], [P], s[INITIALIZING]), reason [after recovery from gateway]]: done applying updated cluster_state
[2012-08-01 14:42:35,446][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[aaanode3], [P], s[INITIALIZING]), reason [after recovery from gateway]]: execute
[2012-08-01 14:42:35,446][DEBUG][cluster.action.shard     ] [node1] applying started shards [[test1][0], node[aaanode3], [P], s[INITIALIZING], [test1][4], node[aaanode3], [P], s[INITIALIZING], [test1][0], node[aaanode3], [P], s[INITIALIZING], [test1][4], node[aaanode3], [P], s[INITIALIZING], [test1][0], node[aaanode3], [P], s[INITIALIZING], [test1][4], node[aaanode3], [P], s[INITIALIZING], [test1][2], node[zzznode2], [P], s[INITIALIZING]], reason [after recovery from gateway]
[2012-08-01 14:42:35,451][TRACE][cluster.service          ] [node1] cluster state updated:
version [8], source [shard-started ([test1][0], node[aaanode3], [P], s[INITIALIZING]), reason [after recovery from gateway]]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[INITIALIZING]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[INITIALIZING]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[INITIALIZING]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]
---- unassigned

[2012-08-01 14:42:35,468][DEBUG][cluster.action.shard     ] [node1] sending shard started for [test1][1], node[zzznode1], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:42:35,468][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][1], node[zzznode1], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:42:35,468][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][3], node[zzznode2], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node1][zzznode1][inet[/192.168.7.132:9300]]]]
[2012-08-01 14:42:35,485][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[aaanode3], [P], s[INITIALIZING]), reason [after recovery from gateway]]: done applying updated cluster_state
[2012-08-01 14:42:35,486][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][4], node[aaanode3], [P], s[INITIALIZING]), reason [after recovery from gateway]]: execute
[2012-08-01 14:42:35,486][DEBUG][cluster.action.shard     ] [node1] applying started shards [[test1][1], node[zzznode1], [R], s[INITIALIZING], [test1][3], node[zzznode2], [R], s[INITIALIZING]], reason [after recovery from gateway]
[2012-08-01 14:42:35,487][TRACE][cluster.service          ] [node1] cluster state updated:
version [9], source [shard-started ([test1][4], node[aaanode3], [P], s[INITIALIZING]), reason [after recovery from gateway]]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[INITIALIZING]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[INITIALIZING]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]
---- unassigned

[2012-08-01 14:42:35,505][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][4], node[aaanode3], [P], s[INITIALIZING]), reason [after recovery from gateway]]: done applying updated cluster_state
[2012-08-01 14:42:35,505][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[aaanode3], [P], s[INITIALIZING]), reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]]: execute
[2012-08-01 14:42:35,505][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[aaanode3], [P], s[INITIALIZING]), reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]]: no change in cluster_state
[2012-08-01 14:42:35,505][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][4], node[aaanode3], [P], s[INITIALIZING]), reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]]: execute
[2012-08-01 14:42:35,505][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][4], node[aaanode3], [P], s[INITIALIZING]), reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]]: no change in cluster_state
[2012-08-01 14:42:35,505][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[aaanode3], [P], s[INITIALIZING]), reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]]: execute
[2012-08-01 14:42:35,505][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[aaanode3], [P], s[INITIALIZING]), reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]]: no change in cluster_state
[2012-08-01 14:42:35,505][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][4], node[aaanode3], [P], s[INITIALIZING]), reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]]: execute
[2012-08-01 14:42:35,505][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][4], node[aaanode3], [P], s[INITIALIZING]), reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]]: no change in cluster_state
[2012-08-01 14:42:35,505][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][1], node[zzznode2], [P], s[INITIALIZING]), reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]]: execute
[2012-08-01 14:42:35,505][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][1], node[zzznode2], [P], s[INITIALIZING]), reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]]: no change in cluster_state
[2012-08-01 14:42:35,505][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][2], node[zzznode2], [P], s[INITIALIZING]), reason [after recovery from gateway]]: execute
[2012-08-01 14:42:35,505][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][2], node[zzznode2], [P], s[INITIALIZING]), reason [after recovery from gateway]]: no change in cluster_state
[2012-08-01 14:42:35,505][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][1], node[zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: execute
[2012-08-01 14:42:35,506][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][1], node[zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: no change in cluster_state
[2012-08-01 14:42:35,506][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][3], node[zzznode2], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node1][zzznode1][inet[/192.168.7.132:9300]]]]]: execute
[2012-08-01 14:42:35,506][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][3], node[zzznode2], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node1][zzznode1][inet[/192.168.7.132:9300]]]]]: no change in cluster_state
[2012-08-01 14:42:35,509][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][3], node[zzznode2], [R], s[INITIALIZING], reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]
[2012-08-01 14:42:35,509][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][3], node[zzznode2], [R], s[INITIALIZING]), reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]]: execute
[2012-08-01 14:42:35,509][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][3], node[zzznode2], [R], s[INITIALIZING]), reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]]: no change in cluster_state
[2012-08-01 14:42:35,510][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][2], node[aaanode3], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:42:35,510][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][2], node[aaanode3], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: execute
[2012-08-01 14:42:35,510][DEBUG][cluster.action.shard     ] [node1] applying started shards [[test1][2], node[aaanode3], [R], s[INITIALIZING]], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:42:35,511][TRACE][cluster.service          ] [node1] cluster state updated:
version [10], source [shard-started ([test1][2], node[aaanode3], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]
---- unassigned

[2012-08-01 14:42:35,513][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][2], node[aaanode3], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: done applying updated cluster_state
[2012-08-01 14:42:35,523][DEBUG][cluster.action.shard     ] [node1] sending shard started for [test1][4], node[zzznode1], [R], s[INITIALIZING], reason [after recovery (replica) from node [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]]]
[2012-08-01 14:42:35,524][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][4], node[zzznode1], [R], s[INITIALIZING], reason [after recovery (replica) from node [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]]]
[2012-08-01 14:42:35,524][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][4], node[zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]]]]: execute
[2012-08-01 14:42:35,524][DEBUG][cluster.action.shard     ] [node1] applying started shards [[test1][4], node[zzznode1], [R], s[INITIALIZING]], reason [after recovery (replica) from node [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]]]
[2012-08-01 14:42:35,525][TRACE][cluster.service          ] [node1] cluster state updated:
version [11], source [shard-started ([test1][4], node[zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]]]]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]
---- unassigned

[2012-08-01 14:42:35,535][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][4], node[zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]]]]: done applying updated cluster_state
[2012-08-01 14:42:35,535][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][0], node[zzznode2], [R], s[INITIALIZING], reason [after recovery (replica) from node [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]]]
[2012-08-01 14:42:35,536][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[zzznode2], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]]]]: execute
[2012-08-01 14:42:35,536][DEBUG][cluster.action.shard     ] [node1] applying started shards [[test1][0], node[zzznode2], [R], s[INITIALIZING]], reason [after recovery (replica) from node [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]]]
[2012-08-01 14:42:35,537][TRACE][cluster.service          ] [node1] cluster state updated:
version [12], source [shard-started ([test1][0], node[zzznode2], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]]]]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]
---- unassigned

[2012-08-01 14:42:35,538][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[zzznode2], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]]]]: done applying updated cluster_state
[2012-08-01 14:42:35,545][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][0], node[zzznode2], [R], s[INITIALIZING], reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]
[2012-08-01 14:42:35,546][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[zzznode2], [R], s[INITIALIZING]), reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]]: execute
[2012-08-01 14:42:35,546][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[zzznode2], [R], s[INITIALIZING]), reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]]: no change in cluster_state
[2012-08-01 14:42:35,555][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][0], node[zzznode2], [R], s[INITIALIZING], reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]
[2012-08-01 14:42:35,555][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[zzznode2], [R], s[INITIALIZING]), reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]]: execute
[2012-08-01 14:42:35,555][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[zzznode2], [R], s[INITIALIZING]), reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]]: no change in cluster_state
[2012-08-01 14:42:48,218][TRACE][discovery.zen.fd         ] [node1] [node  ] failed to ping [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]], retry [1] out of [2]
org.elasticsearch.transport.ReceiveTimeoutTransportException: [isolatednode][inet[/192.168.7.130:9300]][discovery/zen/fd/ping] request_id [259] timed out after [3001ms]
	at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:347)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:662)
[2012-08-01 14:42:51,220][TRACE][discovery.zen.fd         ] [node1] [node  ] failed to ping [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]], retry [2] out of [2]
org.elasticsearch.transport.ReceiveTimeoutTransportException: [isolatednode][inet[/192.168.7.130:9300]][discovery/zen/fd/ping] request_id [263] timed out after [3001ms]
	at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:347)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:662)
[2012-08-01 14:42:51,220][DEBUG][discovery.zen.fd         ] [node1] [node  ] failed to ping [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]], tried [2] times, each with  maximum [3s] timeout
[2012-08-01 14:42:51,221][DEBUG][cluster.service          ] [node1] processing [zen-disco-node_failed([isolatednode][aaanode3][inet[/192.168.7.130:9300]]), reason failed to ping, tried [2] times, each with maximum [3s] timeout]: execute
[2012-08-01 14:42:51,222][TRACE][cluster.service          ] [node1] cluster state updated:
version [13], source [zen-disco-node_failed([isolatednode][aaanode3][inet[/192.168.7.130:9300]]), reason failed to ping, tried [2] times, each with maximum [3s] timeout]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]

routing_nodes:
-----node_id[aaanode3][X]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]
---- unassigned

[2012-08-01 14:42:51,222][INFO ][cluster.service          ] [node1] removed {[isolatednode][aaanode3][inet[/192.168.7.130:9300]],}, reason: zen-disco-node_failed([isolatednode][aaanode3][inet[/192.168.7.130:9300]]), reason failed to ping, tried [2] times, each with maximum [3s] timeout
[2012-08-01 14:42:51,224][DEBUG][cluster.service          ] [node1] processing [zen-disco-node_failed([isolatednode][aaanode3][inet[/192.168.7.130:9300]]), reason failed to ping, tried [2] times, each with maximum [3s] timeout]: done applying updated cluster_state
[2012-08-01 14:42:51,224][DEBUG][cluster.service          ] [node1] processing [routing-table-updater]: execute
[2012-08-01 14:42:51,231][TRACE][cluster.service          ] [node1] cluster state updated:
version [14], source [routing-table-updater]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[INITIALIZING]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], [R], s[INITIALIZING]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[INITIALIZING]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:42:51,292][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][4], node[zzznode2], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node1][zzznode1][inet[/192.168.7.132:9300]]]]
[2012-08-01 14:42:51,298][DEBUG][cluster.action.shard     ] [node1] sending shard started for [test1][0], node[zzznode1], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:42:51,299][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][0], node[zzznode1], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:42:51,318][DEBUG][cluster.service          ] [node1] processing [routing-table-updater]: done applying updated cluster_state
[2012-08-01 14:42:51,319][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][4], node[zzznode2], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node1][zzznode1][inet[/192.168.7.132:9300]]]]]: execute
[2012-08-01 14:42:51,319][DEBUG][cluster.action.shard     ] [node1] applying started shards [[test1][4], node[zzznode2], [R], s[INITIALIZING], [test1][0], node[zzznode1], [R], s[INITIALIZING]], reason [after recovery (replica) from node [[node1][zzznode1][inet[/192.168.7.132:9300]]]]
[2012-08-01 14:42:51,320][TRACE][cluster.service          ] [node1] cluster state updated:
version [15], source [shard-started ([test1][4], node[zzznode2], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node1][zzznode1][inet[/192.168.7.132:9300]]]]]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], [R], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[INITIALIZING]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:42:51,333][DEBUG][cluster.action.shard     ] [node1] sending shard started for [test1][2], node[zzznode1], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:42:51,333][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][2], node[zzznode1], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:42:51,342][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][4], node[zzznode2], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node1][zzznode1][inet[/192.168.7.132:9300]]]]]: done applying updated cluster_state
[2012-08-01 14:42:51,343][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: execute
[2012-08-01 14:42:51,343][DEBUG][cluster.action.shard     ] [node1] applying started shards [[test1][2], node[zzznode1], [R], s[INITIALIZING]], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:42:51,344][TRACE][cluster.service          ] [node1] cluster state updated:
version [16], source [shard-started ([test1][0], node[zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], [R], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:42:51,353][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: done applying updated cluster_state
[2012-08-01 14:42:51,353][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][2], node[zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: execute
[2012-08-01 14:42:51,354][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][2], node[zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: no change in cluster_state
[2012-08-01 14:44:29,786][TRACE][discovery.zen.ping.multicast] [node1] [16] received ping_request from [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]], sending ping_response{target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]], cluster_name[splitbrain]}
[2012-08-01 14:44:31,291][DEBUG][cluster.service          ] [node1] processing [zen-disco-receive(join from node[[isolatednode][aaanode3][inet[/192.168.7.130:9300]]])]: execute
[2012-08-01 14:44:31,292][TRACE][cluster.service          ] [node1] cluster state updated:
version [17], source [zen-disco-receive(join from node[[isolatednode][aaanode3][inet[/192.168.7.130:9300]]])]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], [R], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:44:31,292][INFO ][cluster.service          ] [node1] added {[isolatednode][aaanode3][inet[/192.168.7.130:9300]],}, reason: zen-disco-receive(join from node[[isolatednode][aaanode3][inet[/192.168.7.130:9300]]])
[2012-08-01 14:44:31,293][DEBUG][cluster.service          ] [node1] processing [zen-disco-receive(join from node[[isolatednode][aaanode3][inet[/192.168.7.130:9300]]])]: done applying updated cluster_state
[2012-08-01 14:44:40,433][DEBUG][cluster.service          ] [node1] processing [routing-table-updater]: execute
[2012-08-01 14:44:40,435][TRACE][cluster.service          ] [node1] cluster state updated:
version [18], source [routing-table-updater]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING]
--------[test1][1], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
--------[test1][1], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:44:40,459][DEBUG][cluster.service          ] [node1] processing [routing-table-updater]: done applying updated cluster_state
[2012-08-01 14:44:40,524][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][0], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:44:40,524][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: execute
[2012-08-01 14:44:40,525][DEBUG][cluster.action.shard     ] [node1] applying started shards [[test1][0], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING]], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:44:40,526][TRACE][cluster.service          ] [node1] cluster state updated:
version [19], source [shard-started ([test1][0], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[aaanode3], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], relocating [aaanode3], [P], s[RELOCATING]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], relocating [aaanode3], [P], s[RELOCATING]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [R], s[STARTED]
--------[test1][1], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING]
--------[test1][2], node[aaanode3], relocating [zzznode2], [P], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:44:40,543][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][1], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:44:40,551][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: done applying updated cluster_state
[2012-08-01 14:44:40,552][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][1], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: execute
[2012-08-01 14:44:40,552][DEBUG][cluster.action.shard     ] [node1] applying started shards [[test1][1], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING]], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:44:40,553][TRACE][cluster.service          ] [node1] cluster state updated:
version [20], source [shard-started ([test1][1], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[aaanode3], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], relocating [aaanode3], [P], s[RELOCATING]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], relocating [aaanode3], [P], s[RELOCATING]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [R], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[aaanode3], relocating [zzznode2], [P], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:44:40,558][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][1], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: done applying updated cluster_state
[2012-08-01 14:44:40,577][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][2], node[aaanode3], relocating [zzznode2], [P], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:44:40,578][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][2], node[aaanode3], relocating [zzznode2], [P], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: execute
[2012-08-01 14:44:40,578][DEBUG][cluster.action.shard     ] [node1] applying started shards [[test1][2], node[aaanode3], relocating [zzznode2], [P], s[INITIALIZING]], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:44:40,579][TRACE][cluster.service          ] [node1] cluster state updated:
version [21], source [shard-started ([test1][2], node[aaanode3], relocating [zzznode2], [P], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]
nodes: 
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[aaanode3], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [R], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:44:40,588][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][2], node[aaanode3], relocating [zzznode2], [P], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: done applying updated cluster_state
[2012-08-01 14:45:12,347][TRACE][discovery.zen.fd         ] [node1] [node  ] failed to ping [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]], retry [1] out of [2]
org.elasticsearch.transport.ReceiveTimeoutTransportException: [isolatednode][inet[/192.168.7.130:9300]][discovery/zen/fd/ping] request_id [490] timed out after [3000ms]
	at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:347)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:662)
[2012-08-01 14:45:15,349][TRACE][discovery.zen.fd         ] [node1] [node  ] failed to ping [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]], retry [2] out of [2]
org.elasticsearch.transport.ReceiveTimeoutTransportException: [isolatednode][inet[/192.168.7.130:9300]][discovery/zen/fd/ping] request_id [494] timed out after [3000ms]
	at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:347)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:662)
[2012-08-01 14:45:15,349][DEBUG][discovery.zen.fd         ] [node1] [node  ] failed to ping [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]], tried [2] times, each with  maximum [3s] timeout
[2012-08-01 14:45:15,350][DEBUG][cluster.service          ] [node1] processing [zen-disco-node_failed([isolatednode][aaanode3][inet[/192.168.7.130:9300]]), reason failed to ping, tried [2] times, each with maximum [3s] timeout]: execute
[2012-08-01 14:45:15,350][TRACE][cluster.service          ] [node1] cluster state updated:
version [22], source [zen-disco-node_failed([isolatednode][aaanode3][inet[/192.168.7.130:9300]]), reason failed to ping, tried [2] times, each with maximum [3s] timeout]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[aaanode3], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[aaanode3][X]
--------[test1][0], node[aaanode3], [R], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:45:15,351][INFO ][cluster.service          ] [node1] removed {[isolatednode][aaanode3][inet[/192.168.7.130:9300]],}, reason: zen-disco-node_failed([isolatednode][aaanode3][inet[/192.168.7.130:9300]]), reason failed to ping, tried [2] times, each with maximum [3s] timeout
[2012-08-01 14:45:15,351][DEBUG][cluster.service          ] [node1] processing [zen-disco-node_failed([isolatednode][aaanode3][inet[/192.168.7.130:9300]]), reason failed to ping, tried [2] times, each with maximum [3s] timeout]: done applying updated cluster_state
[2012-08-01 14:45:15,352][DEBUG][cluster.service          ] [node1] processing [routing-table-updater]: execute
[2012-08-01 14:45:15,360][TRACE][cluster.service          ] [node1] cluster state updated:
version [23], source [routing-table-updater]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [R], s[INITIALIZING]
--------[test1][2], node[zzznode1], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [R], s[INITIALIZING]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], [R], s[INITIALIZING]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
--------[test1][2], node[zzznode1], [P], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:45:15,418][DEBUG][cluster.action.shard     ] [node1] sending shard started for [test1][0], node[zzznode1], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:45:15,419][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][2], node[zzznode2], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node1][zzznode1][inet[/192.168.7.132:9300]]]]
[2012-08-01 14:45:15,419][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][0], node[zzznode1], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:45:15,435][DEBUG][cluster.service          ] [node1] processing [routing-table-updater]: done applying updated cluster_state
[2012-08-01 14:45:15,435][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][2], node[zzznode2], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node1][zzznode1][inet[/192.168.7.132:9300]]]]]: execute
[2012-08-01 14:45:15,435][DEBUG][cluster.action.shard     ] [node1] applying started shards [[test1][2], node[zzznode2], [R], s[INITIALIZING], [test1][0], node[zzznode1], [R], s[INITIALIZING]], reason [after recovery (replica) from node [[node1][zzznode1][inet[/192.168.7.132:9300]]]]
[2012-08-01 14:45:15,436][TRACE][cluster.service          ] [node1] cluster state updated:
version [24], source [shard-started ([test1][2], node[zzznode2], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node1][zzznode1][inet[/192.168.7.132:9300]]]]]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [R], s[STARTED]
--------[test1][2], node[zzznode1], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], [R], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
--------[test1][2], node[zzznode1], [P], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:45:15,448][DEBUG][cluster.action.shard     ] [node1] sending shard started for [test1][1], node[zzznode1], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:45:15,448][DEBUG][cluster.action.shard     ] [node1] received shard started for [test1][1], node[zzznode1], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:45:15,460][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][2], node[zzznode2], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node1][zzznode1][inet[/192.168.7.132:9300]]]]]: done applying updated cluster_state
[2012-08-01 14:45:15,460][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: execute
[2012-08-01 14:45:15,460][DEBUG][cluster.action.shard     ] [node1] applying started shards [[test1][1], node[zzznode1], [R], s[INITIALIZING]], reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]
[2012-08-01 14:45:15,461][TRACE][cluster.service          ] [node1] cluster state updated:
version [25], source [shard-started ([test1][0], node[zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], local, master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [R], s[STARTED]
--------[test1][2], node[zzznode1], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], [R], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][2], node[zzznode1], [P], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:45:15,471][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][0], node[zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: done applying updated cluster_state
[2012-08-01 14:45:15,471][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][1], node[zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: execute
[2012-08-01 14:45:15,471][DEBUG][cluster.service          ] [node1] processing [shard-started ([test1][1], node[zzznode1], [R], s[INITIALIZING]), reason [after recovery (replica) from node [[node2][zzznode2][inet[/192.168.7.132:9301]]]]]: no change in cluster_state
[2012-08-01 14:48:40,532][DEBUG][cluster.service          ] [node1] processing [zen-disco-node_left([node2][zzznode2][inet[/192.168.7.132:9301]])]: execute
[2012-08-01 14:48:40,532][WARN ][discovery.zen            ] [node1] not enough master nodes, current nodes: {[node1][zzznode1][inet[/192.168.7.132:9300]],}
[2012-08-01 14:48:40,532][TRACE][cluster.service          ] [node1] cluster state updated:
version [25], source [zen-disco-node_left([node2][zzznode2][inet[/192.168.7.132:9301]])]
nodes: 
   [node1][zzznode1][inet[/192.168.7.132:9300]], local
routing_table:
routing_nodes:
---- unassigned

[2012-08-01 14:48:40,533][INFO ][cluster.service          ] [node1] removed {[node2][zzznode2][inet[/192.168.7.132:9301]],}, reason: zen-disco-node_left([node2][zzznode2][inet[/192.168.7.132:9301]])
[2012-08-01 14:48:40,533][TRACE][discovery.zen.ping.multicast] [node1] [3] sending ping request
[2012-08-01 14:48:40,551][DEBUG][cluster.service          ] [node1] processing [zen-disco-node_left([node2][zzznode2][inet[/192.168.7.132:9301]])]: done applying updated cluster_state
[2012-08-01 14:48:42,034][TRACE][discovery.zen.ping.multicast] [node1] [3] sending ping request
[2012-08-01 14:48:42,193][INFO ][node                     ] [node1] {0.20.0.Beta1-SNAPSHOT}[32215]: stopping ...
[2012-08-01 14:48:42,209][INFO ][node                     ] [node1] {0.20.0.Beta1-SNAPSHOT}[32215]: stopped
[2012-08-01 14:48:42,209][INFO ][node                     ] [node1] {0.20.0.Beta1-SNAPSHOT}[32215]: closing ...
[2012-08-01 14:48:42,216][INFO ][node                     ] [node1] {0.20.0.Beta1-SNAPSHOT}[32215]: closed

splitbrain-node2.log

[2012-08-01 14:40:06,397][INFO ][node                     ] [node2] {0.20.0.Beta1-SNAPSHOT}[32250]: initializing ...
[2012-08-01 14:40:06,403][INFO ][plugins                  ] [node2] loaded [], sites []
[2012-08-01 14:40:07,471][DEBUG][discovery.zen.ping.multicast] [node2] using group [224.2.2.4], with port [54328], ttl [3], and address [null]
[2012-08-01 14:40:07,474][DEBUG][discovery.zen.ping.unicast] [node2] using initial hosts [], with concurrent_connects [10]
[2012-08-01 14:40:07,475][DEBUG][discovery.zen            ] [node2] using ping.timeout [3s], master_election.filter_client [true], master_election.filter_data [false]
[2012-08-01 14:40:07,480][DEBUG][discovery.zen.elect      ] [node2] using minimum_master_nodes [2]
[2012-08-01 14:40:07,481][DEBUG][discovery.zen.fd         ] [node2] [master] uses ping_interval [1s], ping_timeout [3s], ping_retries [2]
[2012-08-01 14:40:07,484][DEBUG][discovery.zen.fd         ] [node2] [node  ] uses ping_interval [1s], ping_timeout [3s], ping_retries [2]
[2012-08-01 14:40:08,252][DEBUG][cluster.routing.allocation.decider] [node2] using node_concurrent_recoveries [2], node_initial_primaries_recoveries [4]
[2012-08-01 14:40:08,253][DEBUG][cluster.routing.allocation.decider] [node2] using [cluster.routing.allocation.allow_rebalance] with [indices_all_active]
[2012-08-01 14:40:08,254][DEBUG][cluster.routing.allocation.decider] [node2] using [cluster_concurrent_rebalance] with [2]
[2012-08-01 14:40:08,418][INFO ][node                     ] [node2] {0.20.0.Beta1-SNAPSHOT}[32250]: initialized
[2012-08-01 14:40:08,418][INFO ][node                     ] [node2] {0.20.0.Beta1-SNAPSHOT}[32250]: starting ...
[2012-08-01 14:40:08,484][INFO ][transport                ] [node2] bound_address {inet[/0:0:0:0:0:0:0:0:9301]}, publish_address {inet[/192.168.7.132:9301]}
[2012-08-01 14:40:08,494][TRACE][discovery                ] [node2] waiting for 30s for the initial state to be set by the discovery
[2012-08-01 14:40:08,505][TRACE][discovery.zen.ping.multicast] [node2] [1] sending ping request
[2012-08-01 14:40:08,559][TRACE][discovery.zen.ping.multicast] [node2] [1] received ping_response{target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [null], cluster_name[splitbrain]}
[2012-08-01 14:40:08,915][TRACE][discovery.zen.ping.multicast] [node2] [2] received ping_request from [[node1][zzznode1][inet[/192.168.7.132:9300]]], sending ping_response{target [[node2][zzznode2][inet[/192.168.7.132:9301]]], master [null], cluster_name[splitbrain]}
[2012-08-01 14:40:10,007][TRACE][discovery.zen.ping.multicast] [node2] [1] sending ping request
[2012-08-01 14:40:10,008][TRACE][discovery.zen.ping.multicast] [node2] [1] received ping_response{target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [null], cluster_name[splitbrain]}
[2012-08-01 14:40:11,511][TRACE][discovery.zen            ] [node2] full ping responses:
	--> target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [null]
[2012-08-01 14:40:11,511][DEBUG][discovery.zen            ] [node2] filtered ping responses: (filter_client[true], filter_data[false])
	--> target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [null]
[2012-08-01 14:40:11,512][TRACE][discovery.zen.ping.multicast] [node2] [2] sending ping request
[2012-08-01 14:40:11,513][TRACE][discovery.zen.ping.multicast] [node2] [2] received ping_response{target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]], cluster_name[splitbrain]}
[2012-08-01 14:40:13,012][TRACE][discovery.zen.ping.multicast] [node2] [2] sending ping request
[2012-08-01 14:40:13,013][TRACE][discovery.zen.ping.multicast] [node2] [2] received ping_response{target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]], cluster_name[splitbrain]}
[2012-08-01 14:40:14,512][TRACE][discovery.zen            ] [node2] full ping responses:
	--> target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]]
[2012-08-01 14:40:14,513][DEBUG][discovery.zen            ] [node2] filtered ping responses: (filter_client[true], filter_data[false])
	--> target [[node1][zzznode1][inet[/192.168.7.132:9300]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]]
[2012-08-01 14:40:14,524][DEBUG][discovery.zen.fd         ] [node2] [master] starting fault detection against master [[node1][zzznode1][inet[/192.168.7.132:9300]]], reason [initial_join]
[2012-08-01 14:40:14,534][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:40:14,534][DEBUG][cluster.service          ] [node2] got first state from fresh master [zzznode1]
[2012-08-01 14:40:14,535][TRACE][cluster.service          ] [node2] cluster state updated:
version [2], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
routing_nodes:
---- unassigned

[2012-08-01 14:40:14,536][INFO ][cluster.service          ] [node2] detected_master [node1][zzznode1][inet[/192.168.7.132:9300]], added {[node1][zzznode1][inet[/192.168.7.132:9300]],}, reason: zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])
[2012-08-01 14:40:14,559][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:40:14,559][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:40:14,559][TRACE][discovery                ] [node2] initial state set from discovery
[2012-08-01 14:40:14,559][TRACE][cluster.service          ] [node2] cluster state updated:
version [3], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
routing_nodes:
---- unassigned

[2012-08-01 14:40:14,559][INFO ][discovery                ] [node2] splitbrain/zzznode2
[2012-08-01 14:40:14,565][INFO ][http                     ] [node2] bound_address {inet[/0:0:0:0:0:0:0:0:9201]}, publish_address {inet[/192.168.7.132:9201]}
[2012-08-01 14:40:14,565][INFO ][node                     ] [node2] {0.20.0.Beta1-SNAPSHOT}[32250]: started
[2012-08-01 14:40:14,634][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:14,139][TRACE][discovery.zen.ping.multicast] [node2] [1] received ping_request from [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]], sending ping_response{target [[node2][zzznode2][inet[/192.168.7.132:9301]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]], cluster_name[splitbrain]}
[2012-08-01 14:42:15,642][TRACE][discovery.zen.ping.multicast] [node2] [1] received ping_request from [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]], sending ping_response{target [[node2][zzznode2][inet[/192.168.7.132:9301]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]], cluster_name[splitbrain]}
[2012-08-01 14:42:17,177][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:17,177][TRACE][cluster.service          ] [node2] cluster state updated:
version [4], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
routing_nodes:
---- unassigned

[2012-08-01 14:42:17,177][INFO ][cluster.service          ] [node2] added {[isolatednode][aaanode3][inet[/192.168.7.130:9300]],}, reason: zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])
[2012-08-01 14:42:17,177][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:35,036][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:35,037][TRACE][cluster.service          ] [node2] cluster state updated:
version [5], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][0], node[null], [R], s[UNASSIGNED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[INITIALIZING]
--------[test1][1], node[null], [R], s[UNASSIGNED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
--------[test1][2], node[null], [R], s[UNASSIGNED]
----shard_id [test1][3]
--------[test1][3], node[zzznode1], [P], s[INITIALIZING]
--------[test1][3], node[null], [R], s[UNASSIGNED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[null], [R], s[UNASSIGNED]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
-----node_id[zzznode2][V]
--------[test1][1], node[zzznode2], [P], s[INITIALIZING]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][3], node[zzznode1], [P], s[INITIALIZING]
---- unassigned
--------[test1][0], node[null], [R], s[UNASSIGNED]
--------[test1][1], node[null], [R], s[UNASSIGNED]
--------[test1][2], node[null], [R], s[UNASSIGNED]
--------[test1][3], node[null], [R], s[UNASSIGNED]
--------[test1][4], node[null], [R], s[UNASSIGNED]

[2012-08-01 14:42:35,392][DEBUG][cluster.action.shard     ] [node2] sending shard started for [test1][1], node[zzznode2], [P], s[INITIALIZING], reason [after recovery from gateway]
[2012-08-01 14:42:35,403][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:35,404][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:35,404][TRACE][cluster.service          ] [node2] cluster state updated:
version [6], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][0], node[null], [R], s[UNASSIGNED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[INITIALIZING]
--------[test1][1], node[null], [R], s[UNASSIGNED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
--------[test1][2], node[null], [R], s[UNASSIGNED]
----shard_id [test1][3]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][3], node[null], [R], s[UNASSIGNED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[null], [R], s[UNASSIGNED]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
-----node_id[zzznode2][V]
--------[test1][1], node[zzznode2], [P], s[INITIALIZING]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][3], node[zzznode1], [P], s[STARTED]
---- unassigned
--------[test1][0], node[null], [R], s[UNASSIGNED]
--------[test1][1], node[null], [R], s[UNASSIGNED]
--------[test1][2], node[null], [R], s[UNASSIGNED]
--------[test1][3], node[null], [R], s[UNASSIGNED]
--------[test1][4], node[null], [R], s[UNASSIGNED]

[2012-08-01 14:42:35,405][DEBUG][cluster.action.shard     ] [node2] sending shard started for [test1][1], node[zzznode2], [P], s[INITIALIZING], reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]
[2012-08-01 14:42:35,405][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:35,405][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:35,406][TRACE][cluster.service          ] [node2] cluster state updated:
version [7], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][0], node[null], [R], s[UNASSIGNED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
--------[test1][2], node[null], [R], s[UNASSIGNED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[INITIALIZING]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[null], [R], s[UNASSIGNED]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[INITIALIZING]
--------[test1][4], node[aaanode3], [P], s[INITIALIZING]
-----node_id[zzznode2][V]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[INITIALIZING]
--------[test1][3], node[zzznode2], [R], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
--------[test1][3], node[zzznode1], [P], s[STARTED]
---- unassigned
--------[test1][0], node[null], [R], s[UNASSIGNED]
--------[test1][2], node[null], [R], s[UNASSIGNED]
--------[test1][4], node[null], [R], s[UNASSIGNED]

[2012-08-01 14:42:35,412][DEBUG][cluster.action.shard     ] [node2] sending shard started for [test1][2], node[zzznode2], [P], s[INITIALIZING], reason [after recovery from gateway]
[2012-08-01 14:42:35,467][DEBUG][cluster.action.shard     ] [node2] sending shard started for [test1][3], node[zzznode2], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node1][zzznode1][inet[/192.168.7.132:9300]]]]
[2012-08-01 14:42:35,469][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:35,469][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:35,469][TRACE][cluster.service          ] [node2] cluster state updated:
version [8], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[INITIALIZING]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[INITIALIZING]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[INITIALIZING]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]
---- unassigned

[2012-08-01 14:42:35,508][DEBUG][cluster.action.shard     ] [node2] sending shard started for [test1][3], node[zzznode2], [R], s[INITIALIZING], reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]
[2012-08-01 14:42:35,518][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:35,518][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:35,518][TRACE][cluster.service          ] [node2] cluster state updated:
version [9], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[INITIALIZING]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[INITIALIZING]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]
---- unassigned

[2012-08-01 14:42:35,534][DEBUG][cluster.action.shard     ] [node2] sending shard started for [test1][0], node[zzznode2], [R], s[INITIALIZING], reason [after recovery (replica) from node [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]]]
[2012-08-01 14:42:35,543][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:35,544][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:35,544][TRACE][cluster.service          ] [node2] cluster state updated:
version [10], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[INITIALIZING]
---- unassigned

[2012-08-01 14:42:35,544][DEBUG][cluster.action.shard     ] [node2] sending shard started for [test1][0], node[zzznode2], [R], s[INITIALIZING], reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]
[2012-08-01 14:42:35,553][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:35,554][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:35,554][TRACE][cluster.service          ] [node2] cluster state updated:
version [11], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[INITIALIZING]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]
---- unassigned

[2012-08-01 14:42:35,554][DEBUG][cluster.action.shard     ] [node2] sending shard started for [test1][0], node[zzznode2], [R], s[INITIALIZING], reason [master [node1][zzznode1][inet[/192.168.7.132:9300]] marked shard as initializing, but shard already started, mark shard as started]
[2012-08-01 14:42:35,555][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:35,555][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:35,555][TRACE][cluster.service          ] [node2] cluster state updated:
version [12], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]

routing_nodes:
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]
---- unassigned

[2012-08-01 14:42:35,564][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:51,224][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:51,224][TRACE][cluster.service          ] [node2] cluster state updated:
version [13], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][0], node[zzznode2], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[aaanode3], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]

routing_nodes:
-----node_id[aaanode3][X]
--------[test1][0], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[aaanode3], [R], s[STARTED]
--------[test1][4], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [R], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [R], s[STARTED]
---- unassigned

[2012-08-01 14:42:51,225][INFO ][cluster.service          ] [node2] removed {[isolatednode][aaanode3][inet[/192.168.7.130:9300]],}, reason: zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])
[2012-08-01 14:42:51,225][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:51,233][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:51,233][TRACE][cluster.service          ] [node2] cluster state updated:
version [14], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[INITIALIZING]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], [R], s[INITIALIZING]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[INITIALIZING]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:42:51,291][DEBUG][cluster.action.shard     ] [node2] sending shard started for [test1][4], node[zzznode2], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node1][zzznode1][inet[/192.168.7.132:9300]]]]
[2012-08-01 14:42:51,291][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:51,321][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:51,322][TRACE][cluster.service          ] [node2] cluster state updated:
version [15], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], [R], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[INITIALIZING]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:42:51,342][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:42:51,346][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:42:51,346][TRACE][cluster.service          ] [node2] cluster state updated:
version [16], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], [R], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:42:51,359][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:44:29,786][TRACE][discovery.zen.ping.multicast] [node2] [16] received ping_request from [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]], sending ping_response{target [[node2][zzznode2][inet[/192.168.7.132:9301]]], master [[node1][zzznode1][inet[/192.168.7.132:9300]]], cluster_name[splitbrain]}
[2012-08-01 14:44:31,294][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:44:31,295][TRACE][cluster.service          ] [node2] cluster state updated:
version [17], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], [R], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:44:31,295][INFO ][cluster.service          ] [node2] added {[isolatednode][aaanode3][inet[/192.168.7.130:9300]],}, reason: zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])
[2012-08-01 14:44:31,296][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:44:40,437][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:44:40,437][TRACE][cluster.service          ] [node2] cluster state updated:
version [18], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING]
--------[test1][1], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
--------[test1][1], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:44:40,464][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:44:40,527][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:44:40,528][TRACE][cluster.service          ] [node2] cluster state updated:
version [19], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[aaanode3], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], relocating [aaanode3], [P], s[RELOCATING]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], relocating [aaanode3], [P], s[RELOCATING]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [R], s[STARTED]
--------[test1][1], node[aaanode3], relocating [zzznode1], [R], s[INITIALIZING]
--------[test1][2], node[aaanode3], relocating [zzznode2], [P], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][1], node[zzznode1], relocating [aaanode3], [R], s[RELOCATING]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:44:40,543][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:44:40,555][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:44:40,555][TRACE][cluster.service          ] [node2] cluster state updated:
version [20], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[aaanode3], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], relocating [aaanode3], [P], s[RELOCATING]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], relocating [aaanode3], [P], s[RELOCATING]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [R], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[aaanode3], relocating [zzznode2], [P], s[INITIALIZING]
-----node_id[zzznode1][V]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:44:40,563][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:44:40,580][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:44:40,581][TRACE][cluster.service          ] [node2] cluster state updated:
version [21], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [isolatednode][aaanode3][inet[/192.168.7.130:9300]]
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[aaanode3], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[aaanode3][V]
--------[test1][0], node[aaanode3], [R], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:44:40,597][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:45:15,352][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:45:15,352][TRACE][cluster.service          ] [node2] cluster state updated:
version [22], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[aaanode3], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[aaanode3], [P], s[STARTED]
--------[test1][2], node[zzznode1], [R], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[aaanode3][X]
--------[test1][0], node[aaanode3], [R], s[STARTED]
--------[test1][1], node[aaanode3], [R], s[STARTED]
--------[test1][2], node[aaanode3], [P], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][2], node[zzznode1], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:45:15,353][INFO ][cluster.service          ] [node2] removed {[isolatednode][aaanode3][inet[/192.168.7.130:9300]],}, reason: zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])
[2012-08-01 14:45:15,353][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:45:15,362][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:45:15,362][TRACE][cluster.service          ] [node2] cluster state updated:
version [23], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [R], s[INITIALIZING]
--------[test1][2], node[zzznode1], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [R], s[INITIALIZING]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], [R], s[INITIALIZING]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
--------[test1][2], node[zzznode1], [P], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:45:15,417][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:45:15,418][DEBUG][cluster.action.shard     ] [node2] sending shard started for [test1][2], node[zzznode2], [R], s[INITIALIZING], reason [after recovery (replica) from node [[node1][zzznode1][inet[/192.168.7.132:9300]]]]
[2012-08-01 14:45:15,438][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:45:15,438][TRACE][cluster.service          ] [node2] cluster state updated:
version [24], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [R], s[STARTED]
--------[test1][2], node[zzznode1], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], [R], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[INITIALIZING]
--------[test1][2], node[zzznode1], [P], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:45:15,460][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:45:15,463][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: execute
[2012-08-01 14:45:15,463][TRACE][cluster.service          ] [node2] cluster state updated:
version [25], source [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]
nodes: 
   [node2][zzznode2][inet[/192.168.7.132:9301]], local
   [node1][zzznode1][inet[/192.168.7.132:9300]], master
routing_table:
-- index [test1]
----shard_id [test1][0]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][0], node[zzznode1], [R], s[STARTED]
----shard_id [test1][1]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
----shard_id [test1][2]
--------[test1][2], node[zzznode2], [R], s[STARTED]
--------[test1][2], node[zzznode1], [P], s[STARTED]
----shard_id [test1][3]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
----shard_id [test1][4]
--------[test1][4], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]

routing_nodes:
-----node_id[zzznode2][V]
--------[test1][0], node[zzznode2], [P], s[STARTED]
--------[test1][1], node[zzznode2], [P], s[STARTED]
--------[test1][2], node[zzznode2], [R], s[STARTED]
--------[test1][3], node[zzznode2], [R], s[STARTED]
--------[test1][4], node[zzznode2], [R], s[STARTED]
-----node_id[zzznode1][V]
--------[test1][0], node[zzznode1], [R], s[STARTED]
--------[test1][1], node[zzznode1], [R], s[STARTED]
--------[test1][2], node[zzznode1], [P], s[STARTED]
--------[test1][3], node[zzznode1], [P], s[STARTED]
--------[test1][4], node[zzznode1], [P], s[STARTED]
---- unassigned

[2012-08-01 14:45:15,477][DEBUG][cluster.service          ] [node2] processing [zen-disco-receive(from master [[node1][zzznode1][inet[/192.168.7.132:9300]]])]: done applying updated cluster_state
[2012-08-01 14:48:38,456][INFO ][discovery.zen            ] [node2] master_left [[isolatednode][aaanode3][inet[/192.168.7.130:9300]]], reason [shut_down]
[2012-08-01 14:48:38,459][DEBUG][cluster.service          ] [node2] processing [zen-disco-master_failed ([isolatednode][aaanode3][inet[/192.168.7.130:9300]])]: execute
[2012-08-01 14:48:38,459][DEBUG][cluster.service          ] [node2] processing [zen-disco-master_failed ([isolatednode][aaanode3][inet[/192.168.7.130:9300]])]: no change in cluster_state
[2012-08-01 14:48:40,505][INFO ][node                     ] [node2] {0.20.0.Beta1-SNAPSHOT}[32250]: stopping ...
[2012-08-01 14:48:40,530][DEBUG][discovery.zen.fd         ] [node2] [master] stopping fault detection against master [[node1][zzznode1][inet[/192.168.7.132:9300]]], reason [zen disco stop]
[2012-08-01 14:48:40,542][INFO ][node                     ] [node2] {0.20.0.Beta1-SNAPSHOT}[32250]: stopped
[2012-08-01 14:48:40,542][INFO ][node                     ] [node2] {0.20.0.Beta1-SNAPSHOT}[32250]: closing ...
[2012-08-01 14:48:40,551][INFO ][node                     ] [node2] {0.20.0.Beta1-SNAPSHOT}[32250]: closed