gistfile1.txt

[2013-02-25 09:09:04,104][TRACE][discovery.zen.ping.multicast] [search7] [104] sending ping request
...
[2013-02-25 09:09:05,104][TRACE][discovery.zen            ] [search7] full ping responses:
        --> target [[search12][gxTetOKXQTaJ82k_3w4pMQ][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search9][qPpWpsS0SoG_nQdezJsECA][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search6][z70EGYtFTXGttDDQdtqrbQ][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search10][WTkt_bFKQmmc6dW3q7-WDQ][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search5][f_48G1mtScmy1rQHz-YaaQ][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search11][QPvZFBO1RTWJ2eEru_rKEw][inet[/ipaddress:9300]]], master [[search11][QPvZFBO1RTWJ2eEru_rKEw][inet[/ipaddress:9300]]]
[2013-02-25 09:09:05,105][DEBUG][discovery.zen            ] [search7] filtered ping responses: (filter_client[true], filter_data[false])
        --> target [[search12][gxTetOKXQTaJ82k_3w4pMQ][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search9][qPpWpsS0SoG_nQdezJsECA][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search6][z70EGYtFTXGttDDQdtqrbQ][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search10][WTkt_bFKQmmc6dW3q7-WDQ][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search5][f_48G1mtScmy1rQHz-YaaQ][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search11][QPvZFBO1RTWJ2eEru_rKEw][inet[/ipaddress:9300]]], master [[search11][QPvZFBO1RTWJ2eEru_rKEw][inet[/ipaddress:9300]]]
[2013-02-25 09:09:05,108][WARN ][discovery.zen            ] [search7] failed to connect to master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]], retrying...
org.elasticsearch.transport.ConnectTransportException: [search8][inet[/ipaddress:9300]] connect_timeout[5s]
        at org.elasticsearch.transport.netty.NettyTransport.connectToChannels(NettyTransport.java:665)
        at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:604)
        at org.elasticsearch.transport.netty.NettyTransport.connectToNode(NettyTransport.java:574)
        at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:127)
        at org.elasticsearch.discovery.zen.ZenDiscovery.innterJoinCluster(ZenDiscovery.java:325)
        at org.elasticsearch.discovery.zen.ZenDiscovery.access$500(ZenDiscovery.java:73)
        at org.elasticsearch.discovery.zen.ZenDiscovery$1.run(ZenDiscovery.java:279)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
        at java.lang.Thread.run(Thread.java:722)
Caused by: java.net.ConnectException: Connection refused
        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:701)
        at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.connect(NioClientSocketPipelineSink.java:490)
        at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.processSelectedKeys(NioClientSocketPipelineSink.java:446)
        at org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink$Boss.run(NioClientSocketPipelineSink.java:359)
        at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
        at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
        ... 3 more

gistfile2.txt

[2013-02-25 09:10:47,308][TRACE][discovery.zen.ping.multicast] [search7] [206] sending ping request
...
[2013-02-25 09:10:47,809][TRACE][discovery.zen            ] [search7] full ping responses:
        --> target [[search12][gxTetOKXQTaJ82k_3w4pMQ][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search9][qPpWpsS0SoG_nQdezJsECA][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search6][z70EGYtFTXGttDDQdtqrbQ][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search10][WTkt_bFKQmmc6dW3q7-WDQ][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search5][f_48G1mtScmy1rQHz-YaaQ][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search11][QPvZFBO1RTWJ2eEru_rKEw][inet[/ipaddress:9300]]], master [[search11][QPvZFBO1RTWJ2eEru_rKEw][inet[/ipaddress:9300]]]
        --> target [[search8][feMQtDFOTs2xyh0xcTXdkA][inet[/ipaddress:9300]]], master [null]
[2013-02-25 09:10:47,809][DEBUG][discovery.zen            ] [search7] filtered ping responses: (filter_client[true], filter_data[false])
        --> target [[search12][gxTetOKXQTaJ82k_3w4pMQ][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search9][qPpWpsS0SoG_nQdezJsECA][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search6][z70EGYtFTXGttDDQdtqrbQ][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search10][WTkt_bFKQmmc6dW3q7-WDQ][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search5][f_48G1mtScmy1rQHz-YaaQ][inet[/ipaddress:9300]]], master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]]
        --> target [[search11][QPvZFBO1RTWJ2eEru_rKEw][inet[/ipaddress:9300]]], master [[search11][QPvZFBO1RTWJ2eEru_rKEw][inet[/ipaddress:9300]]]
        --> target [[search8][feMQtDFOTs2xyh0xcTXdkA][inet[/ipaddress:9300]]], master [null]
[2013-02-25 09:10:47,816][INFO ][discovery.zen            ] [search7] failed to send join request to master [[search8][GYhoDKLWRFOCy7KtUgVVQg][inet[/ipaddress:9300]]], reason [org.elasticsearch.transport.RemoteTransportException: [search8][inet[/ipaddress:9300]][discovery/zen/join]; org.elasticsearch.ElasticSearchIllegalStateException: Node [[search8][feMQtDFOTs2xyh0xcTXdkA][inet[/ipaddress:9300]]] not master for join request from [[search7][4lEhStfwQDKuHpGvHeU-hQ][inet[/ipaddress:9300]]]]
[2013-02-25 09:10:47,816][TRACE][discovery.zen            ] [search7] detailed failed reason
org.elasticsearch.transport.RemoteTransportException: [search8][inet[/ipaddress:9300]][discovery/zen/join]
Caused by: org.elasticsearch.ElasticSearchIllegalStateException: Node [[search8][feMQtDFOTs2xyh0xcTXdkA][inet[/ipaddress:9300]]] not master for join request from [[search7][4lEhStfwQDKuHpGvHeU-hQ][inet[/ipaddress:9300]]]
        at org.elasticsearch.discovery.zen.ZenDiscovery.handleJoinRequest(ZenDiscovery.java:582)
        at org.elasticsearch.discovery.zen.ZenDiscovery.access$1900(ZenDiscovery.java:73)
        at org.elasticsearch.discovery.zen.ZenDiscovery$MembershipListener.onJoin(ZenDiscovery.java:735)
        at org.elasticsearch.discovery.zen.membership.MembershipAction$JoinRequestRequestHandler.messageReceived(MembershipAction.java:164)
        at org.elasticsearch.discovery.zen.membership.MembershipAction$JoinRequestRequestHandler.messageReceived(MembershipAction.java:153)
        at org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:268)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
        at java.lang.Thread.run(Thread.java:722)