Created
April 5, 2015 12:38
-
-
Save noniperi/7ba2fafe13676a1d027d to your computer and use it in GitHub Desktop.
cluster_logs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ####################### | |
| Client node: | |
| ####################### | |
| [2015-04-05 11:16:32,468][INFO ][cluster.service ] [client_i-b4e65648] removed {[data_i-a16d184e][7qFiJh-bRNySd6B4cVEg9Q][ip-1-1-1-128][inet[/1.1.1.128:9300]]{max_local_storage_nodes=1, zon | |
| e=us-east-1a, master=false},}, reason: zen-disco-receive(from master [[master_i-c452ee38][JNxRhgR7SAO4fImUuFm5kg][ip-1-1-1-70][inet[/1.1.1.70:9300]]{max_local_storage_nodes=1, data=false, zone=us-ea | |
| st-1a, master=true}]) | |
| [2015-04-05 11:17:41,615][INFO ][cluster.service ] [client_i-b4e65648] added {[data_i-a16d184e][7qFiJh-bRNySd6B4cVEg9Q][ip-1-1-1-128][inet[/1.1.1.128:9300]]{max_local_storage_nodes=1, zone= | |
| us-east-1a, master=false},}, reason: zen-disco-receive(from master [[master_i-c452ee38][JNxRhgR7SAO4fImUuFm5kg][ip-1-1-1-70][inet[/1.1.1.70:9300]]{max_local_storage_nodes=1, data=false, zone=us-east | |
| -1a, master=true}]) | |
| ####################### | |
| Data (data_i-a16d184e): | |
| ####################### | |
| [2015-04-05 11:16:48,703][WARN ][discovery.ec2 ] [data_i-a16d184e] master left (reason = do not exists on master, act as master failure), current nodes: {[client_i-051df6f8][taZapiltSTmHrLV087u | |
| Rew][ip-1-1-1-34][inet[/1.1.1.34:9300]]{max_local_storage_nodes=1, data=false, zone=us-east-1d, master=false},[data_i-a16d184e][7qFiJh-bRNySd6B4cVEg9Q][ip-1-1-1-128][inet[ip-1-1-1-128.ec2.in | |
| ternal/1.1.1.128:9300]]{max_local_storage_nodes=1, zone=us-east-1a, master=false},[client_i-53e656af][s2k_rKN0SWy4Ii1n_lxhnw][ip-1-1-1-135][inet[/1.1.1.135:9300]]{max_local_storage_nodes=1, data= | |
| false, zone=us-east-1a, master=false},[data_i-56ec40ac][4vXLZSyQQ9K8nJvyx7jLhw][ip-1-1-10-160][inet[/1.1.1.160:9300]]{max_local_storage_nodes=1, zone=us-east-1d, master=false},[data_i-31d078cb][8R7 | |
| cr-fxR2yIE2R0Hk670A][ip-1-1-1-176][inet[/1.1.1.176:9300]]{max_local_storage_nodes=1, zone=us-east-1d, master=false},[data_i-8f7d0860][8lNFAJk_TdeHrc-JslvLsw][ip-1-1-1-108][inet[/1.1.1.108:93 | |
| 00]]{max_local_storage_nodes=1, zone=us-east-1a, master=false},[master_i-fb699e06][fUaDrFqpR-C7BXIj8Kaf0A][ip-1-1-1-61][inet[/1.1.1.61:9300]]{max_local_storage_nodes=1, data=false, zone=us-east-1d | |
| , master=true},[master_i-684ef294][q2vr4-JiTtOJstN-MzveHA][ip-1-1-1-125][inet[/1.1.1.125:9300]]{max_local_storage_nodes=1, data=false, zone=us-east-1a, master=true},[LE_client_i-cb955637][eFwE5hiqTI | |
| SJt0HEeFpBBg][ip-1-1-1-80][inet[/1.1.1.80:9300]]{max_local_storage_nodes=1, data=false, zone=us-east-1a, master=false},[LE_client_i-56ee02ab][6yGxm87HRZWUArjj2mTpuA][ip-1-1-1-140][inet[/1.1.1 | |
| 4.140:9300]]{max_local_storage_nodes=1, data=false, zone=us-east-1d, master=false},[client_i-b4e65648][GKkGpbnLSayTbjmFngj6Ng][ip-1-1-1-172][inet[/1.1.1.172:9300]]{max_local_storage_nodes=1, data=fa | |
| lse, zone=us-east-1a, master=false},} | |
| [2015-04-05 11:16:48,704][INFO ][cluster.service ] [data_i-a16d184e] removed {[master_i-c452ee38][JNxRhgR7SAO4fImUuFm5kg][ip-1-1-1-70][inet[/1.1.1.70:9300]]{max_local_storage_nodes=1, data= | |
| false, zone=us-east-1a, master=true},}, reason: zen-disco-master_failed ([master_i-c452ee38][JNxRhgR7SAO4fImUuFm5kg][ip-1-1-1-70][inet[/1.1.1.70:9300]]{max_local_storage_nodes=1, data=false, zone=us | |
| -east-1a, master=true}) | |
| [2015-04-05 11:16:48,757][WARN ][cluster.service ] [data_i-a16d184e] failed to notify ClusterStateListener | |
| org.elasticsearch.ElasticsearchIllegalStateException: can't send request to a null node | |
| at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:255) | |
| at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:249) | |
| at org.elasticsearch.cluster.action.index.NodeMappingRefreshAction.nodeMappingRefresh(NodeMappingRefreshAction.java:62) | |
| at org.elasticsearch.indices.cluster.IndicesClusterStateService.applyMappings(IndicesClusterStateService.java:381) | |
| at org.elasticsearch.indices.cluster.IndicesClusterStateService.clusterChanged(IndicesClusterStateService.java:181) | |
| at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:467) | |
| at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:188) | |
| at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:158) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:745) | |
| [2015-04-05 11:16:54,831][ERROR][marvel.agent.exporter ] [data_i-a16d184e] remote target didn't respond with 200 OK response code [503 Service Unavailable]. content: [:) | |
| ^E��error�ClusterBlockException[blocked by: [SERVICE_UNAVAILABLE/2/no master];]��status$^O��] | |
| [2015-04-05 11:17:05,901][ERROR][marvel.agent.exporter ] [data_i-a16d184e] remote target didn't respond with 200 OK response code [503 Service Unavailable]. content: [:) | |
| ^E��error�ClusterBlockException[blocked by: [SERVICE_UNAVAILABLE/2/no master];]��status$^O��] | |
| [2015-04-05 11:17:16,971][ERROR][marvel.agent.exporter ] [data_i-a16d184e] remote target didn't respond with 200 OK response code [503 Service Unavailable]. content: [:) | |
| ^E��error�ClusterBlockException[blocked by: [SERVICE_UNAVAILABLE/2/no master];]��status$^O��] | |
| [2015-04-05 11:17:28,039][ERROR][marvel.agent.exporter ] [data_i-a16d184e] remote target didn't respond with 200 OK response code [503 Service Unavailable]. content: [:) | |
| ^E��error�ClusterBlockException[blocked by: [SERVICE_UNAVAILABLE/2/no master];]��status$^O��] | |
| [2015-04-05 11:17:39,104][ERROR][marvel.agent.exporter ] [data_i-a16d184e] remote target didn't respond with 200 OK response code [503 Service Unavailable]. content: [:) | |
| ^E��error�ClusterBlockException[blocked by: [SERVICE_UNAVAILABLE/2/no master];]��status$^O��] | |
| [2015-04-05 11:17:50,176][ERROR][marvel.agent.exporter ] [data_i-a16d184e] remote target didn't respond with 200 OK response code [503 Service Unavailable]. content: [:) | |
| ^E��error�ClusterBlockException[blocked by: [SERVICE_UNAVAILABLE/2/no master];]��status$^O��] | |
| [2015-04-05 11:17:50,235][INFO ][cluster.service ] [data_i-a16d184e] detected_master [master_i-c452ee38][JNxRhgR7SAO4fImUuFm5kg][ip-1-1-1-70][inet[/1.1.1.70:9300]]{max_local_storage_nodes=1, data=false, zone=us-east-1a, master=true}, added {[master_i-c452ee38][JNxRhgR7SAO4fImUuFm5kg][ip-1-1-1-70][inet[/1.1.1.70:9300]]{max_local_storage_nodes=1, data=false, zone=us-east-1a, master=true},}, reason: zen-disco-receive(from master [[master_i-c452ee38][JNxRhgR7SAO4fImUuFm5kg][ip-1-1-1-70][inet[/1.1.1.70:9300]]{max_local_storage_nodes=1, data=false, zone=us-east-1a, master=true}]) | |
| ####################### | |
| Master (master_i-c452ee38) | |
| ####################### | |
| [2015-04-05 11:14:02,184][DEBUG][action.admin.indices.stats] [master_i-c452ee38] [accounts-metadata-150215][0], node[7qFiJh-bRNySd6B4cVEg9Q], [R], s[STARTED]: failed to execute [org.elasticsearch.action.a | |
| dmin.indices.stats.IndicesStatsRequest@389148b9] | |
| org.elasticsearch.transport.NodeDisconnectedException: [data_i-a16d184e][inet[ip-1-1-1-128.ec2.internal/1.1.1.128:9300]][indices:monitor/stats[s]] disconnected | |
| [2015-04-05 11:14:02,186][DEBUG][action.admin.indices.stats] [master_i-c452ee38] [accounts-metadata-150317][0], node[7qFiJh-bRNySd6B4cVEg9Q], [R], s[STARTED]: failed to execute [org.elasticsearch.action.a | |
| dmin.indices.stats.IndicesStatsRequest@780b16c] | |
| org.elasticsearch.transport.NodeDisconnectedException: [data_i-a16d184e][inet[ip-1-1-1-128.ec2.internal/1.1.1.128:9300]][indices:monitor/stats[s]] disconnected | |
| [2015-04-05 11:14:02,184][DEBUG][action.admin.indices.stats] [master_i-c452ee38] [kibana-xxxx-tmmpwdigdhvsydlhtezimotjlnepzswt-][0], node[7qFiJh-bRNySd6B4cVEg9Q], [R], s[STARTED]: failed to execute [org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest@e594233] | |
| org.elasticsearch.transport.NodeDisconnectedException: [data_i-a16d184e][inet[ip-1-1-1-128.ec2.internal/1.1.1.128:9300]][indices:monitor/stats[s]] disconnected | |
| [2015-04-05 11:14:02,186][DEBUG][action.admin.indices.stats] [master_i-c452ee38] [accounts-metadata-150224][0], node[7qFiJh-bRNySd6B4cVEg9Q], [R], s[STARTED]: failed to execute [org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest@389148b9] | |
| org.elasticsearch.transport.NodeDisconnectedException: [data_i-a16d184e][inet[ip-1-1-1-128.ec2.internal/1.1.1.128:9300]][indices:monitor/stats[s]] disconnected | |
| [2015-04-05 11:14:02,184][DEBUG][action.admin.indices.stats] [master_i-c452ee38] [accounts-metadata-150224][0], node[7qFiJh-bRNySd6B4cVEg9Q], [R], s[STARTED]: failed to execute [org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest@7a88f33e] | |
| ... | |
| ... | |
| <these types of logs are repeated for all indexes> | |
| ... | |
| ... | |
| [2015-04-05 11:14:02,327][DEBUG][action.admin.indices.stats] [master_i-c452ee38] [xxxx-fuaexagzicaqwarkgqdotwqecptkfibn-150403][5], node[7qFiJh-bRNySd6B4cVEg9Q], [R], s[STARTED]: failed to execute [org.el | |
| asticsearch.action.admin.indices.stats.IndicesStatsRequest@79856790] | |
| org.elasticsearch.transport.NodeDisconnectedException: [data_i-a16d184e][inet[ip-1-1-1-128.ec2.internal/1.1.1.128:9300]][indices:monitor/stats[s]] disconnected | |
| [2015-04-05 11:14:02,327][DEBUG][action.admin.indices.stats] [master_i-c452ee38] [xxxx-fuaexagzicaqwarkgqdotwqecptkfibn-150331][7], node[7qFiJh-bRNySd6B4cVEg9Q], [R], s[STARTED]: failed to execute [org.el | |
| asticsearch.action.admin.indices.stats.IndicesStatsRequest@79856790] | |
| org.elasticsearch.transport.NodeDisconnectedException: [data_i-a16d184e][inet[ip-1-1-1-128.ec2.internal/1.1.1.128:9300]][indices:monitor/stats[s]] disconnected | |
| [2015-04-05 11:14:03,187][DEBUG][action.admin.cluster.stats] [master_i-c452ee38] failed to execute on node [7qFiJh-bRNySd6B4cVEg9Q] | |
| org.elasticsearch.transport.SendRequestTransportException: [data_i-a16d184e][inet[ip-1-1-1-128.ec2.internal/1.1.1.128:9300]][cluster:monitor/stats[n]] | |
| at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:284) | |
| at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.start(TransportNodesOperationAction.java:165) | |
| at org.elasticsearch.action.support.nodes.TransportNodesOperationAction$AsyncAction.access$300(TransportNodesOperationAction.java:97) | |
| at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:70) | |
| at org.elasticsearch.action.support.nodes.TransportNodesOperationAction.doExecute(TransportNodesOperationAction.java:43) | |
| at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:75) | |
| at org.elasticsearch.client.node.NodeClusterAdminClient.execute(NodeClusterAdminClient.java:77) | |
| at org.elasticsearch.client.support.AbstractClusterAdminClient.clusterStats(AbstractClusterAdminClient.java:217) | |
| at org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequestBuilder.doExecute(ClusterStatsRequestBuilder.java:37) | |
| at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:91) | |
| at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:65) | |
| at org.elasticsearch.action.ActionRequestBuilder.get(ActionRequestBuilder.java:73) | |
| at org.elasticsearch.marvel.agent.AgentService$ExportingWorker.exportClusterStats(AgentService.java:314) | |
| at org.elasticsearch.marvel.agent.AgentService$ExportingWorker.run(AgentService.java:261) | |
| at java.lang.Thread.run(Thread.java:745) | |
| Caused by: org.elasticsearch.transport.NodeNotConnectedException: [data_i-a16d184e][inet[ip-1-1-1-128.ec2.internal/1.1.1.128:9300]] Node not connected | |
| at org.elasticsearch.transport.netty.NettyTransport.nodeChannel(NettyTransport.java:936) | |
| at org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:629) | |
| at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:270) | |
| ... 14 more | |
| [2015-04-05 11:16:07,147][INFO ][cluster.service ] [master_i-c452ee38] removed {[data_i-a16d184e][7qFiJh-bRNySd6B4cVEg9Q][ip-1-1-1-128][inet[ip-1-1-1-128.ec2.internal/1.1.1.128:9300]]{max_local_storage_nodes=1, zone=us-east-1a, master=false},}, reason: zen-disco-node_failed([data_i-a16d184e][7qFiJh-bRNySd6B4cVEg9Q][ip-1-1-1-128][inet[ip-1-1-1-128.ec2.internal/1.1.1.128:9300]]{max_local_storage_nodes=1, zone=us-east-1a, master=false}), reason transport disconnected | |
| [2015-04-05 11:16:13,604][INFO ][cluster.metadata ] [master_i-c452ee38] [xxxx-jopnbwmknooanqwzxgpybunufztysazs-150405] update_mapping [quievnqmpmrrl20rfrm2j3iu1e] (dynamic) | |
| [2015-04-05 11:16:13,604][INFO ][cluster.metadata ] [master_i-c452ee38] [xxxx-jopnbwmknooanqwzxgpybunufztysazs-150405] update_mapping [hu81qasnmh3jqoqmqg95f8svlc] (dynamic) | |
| [2015-04-05 11:16:13,653][INFO ][cluster.metadata ] [master_i-c452ee38] [xxxx-fuaexagzicaqwarkgqdotwqecptkfibn-150405] update_mapping [json] (dynamic) | |
| [2015-04-05 11:16:13,675][INFO ][cluster.metadata ] [master_i-c452ee38] [xxxx-fuaexagzicaqwarkgqdotwqecptkfibn-150405] update_mapping [json] (dynamic) | |
| [2015-04-05 11:17:16,235][INFO ][cluster.service ] [master_i-c452ee38] added {[data_i-a16d184e][7qFiJh-bRNySd6B4cVEg9Q][ip-1-1-1-128][inet[/1.1.1.128:9300]]{max_local_storage_nodes=1, zone=us-east-1a, master=false},}, reason: zen-disco-receive(join from node[[data_i-a16d184e][7qFiJh-bRNySd6B4cVEg9Q][ip-1-1-1-128][inet[/1.1.1.128:9300]]{max_local_storage_nodes=1, zone=us-east-1a, master=false}]) | |
| [2015-04-05 11:18:58,138][INFO ][cluster.metadata ] [master_i-c452ee38] [xxxx-fuaexagzicaqwarkgqdotwqecptkfibn-150405] update_mapping [json] (dynamic) | |
| [2015-04-05 11:20:32,400][INFO ][cluster.metadata ] [master_i-c452ee38] [xxxx-fuaexagzicaqwarkgqdotwqecptkfibn-150405] update_mapping [json] (dynamic) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment