-
-
Save rafbgarcia/22daef86c55cea91c45ae9fb175671ca to your computer and use it in GitHub Desktop.
Master Node: crate02
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [2022-05-22T23:59:35,707][INFO ][o.e.c.c.Coordinator ] [cratedb01] master node [{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{tj6qgjVFS0C0jZfPoaNnBg}{192.168.80.2}{192.168.80.2:4300}{http_address=192.168.80.2:4200}] failed, restarting discovery | |
| org.elasticsearch.transport.NodeDisconnectedException: [cratedb02][192.168.80.2:4300][disconnected] disconnected | |
| [2022-05-22T23:59:35,775][INFO ][o.e.c.s.ClusterApplierService] [cratedb01] master node changed {previous [{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{tj6qgjVFS0C0jZfPoaNnBg}{192.168.80.2}{192.168.80.2:4300}{http_address=192.168.80.2:4200}], current []}, term: 17, version: 329, reason: becoming candidate: onLeaderFailure | |
| [2022-05-22T23:59:35,847][WARN ][o.e.c.NodeConnectionsService] [cratedb01] failed to connect to {cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{tj6qgjVFS0C0jZfPoaNnBg}{192.168.80.2}{192.168.80.2:4300}{http_address=192.168.80.2:4200} (tried [1] times) | |
| org.elasticsearch.transport.ConnectTransportException: [cratedb02][192.168.80.2:4300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:961) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$3(ActionListener.java:179) ~[crate-server.jar:?] | |
| at io.crate.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:44) ~[crate-shared.jar:?] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] | |
| at io.crate.concurrent.CompletableContext.addListener(CompletableContext.java:47) ~[crate-shared.jar:?] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.addConnectListener(Netty4TcpChannel.java:103) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.TcpTransport.initiateConnection(TcpTransport.java:322) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.TcpTransport.openConnection(TcpTransport.java:286) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.ClusterConnectionManager.internalOpenConnection(ClusterConnectionManager.java:254) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.ClusterConnectionManager.connectToNode(ClusterConnectionManager.java:139) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:309) ~[crate-server.jar:?] | |
| at org.elasticsearch.cluster.NodeConnectionsService$ConnectionTarget$1.doRun(NodeConnectionsService.java:316) ~[crate-server.jar:?] | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?] | |
| at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] | |
| at org.elasticsearch.cluster.NodeConnectionsService.connectToNodes(NodeConnectionsService.java:139) ~[crate-server.jar:?] | |
| at org.elasticsearch.cluster.service.ClusterApplierService.connectToNodesAndWait(ClusterApplierService.java:482) ~[crate-server.jar:?] | |
| at org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:456) ~[crate-server.jar:?] | |
| at org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:416) ~[crate-server.jar:?] | |
| at org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:163) ~[crate-server.jar:?] | |
| at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:259) ~[crate-server.jar:?] | |
| at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:222) ~[crate-server.jar:?] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?] | |
| at java.lang.Thread.run(Thread.java:833) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 192.168.80.2/192.168.80.2:4300 | |
| Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused | |
| at io.netty.channel.unix.Errors.newConnectException0(Errors.java:155) ~[netty-transport-native-unix-common-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:128) ~[netty-transport-native-unix-common-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.unix.Socket.finishConnect(Socket.java:321) ~[netty-transport-native-unix-common-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:710) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:687) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:477) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:385) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) ~[netty-common-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.76.Final.jar:4.1.76.Final] | |
| ... 1 more | |
| [2022-05-22T23:59:35,996][INFO ][o.e.c.s.MasterService ] [cratedb01] elected-as-master ([2] nodes joined)[{cratedb03}{4dJJb88aS767BA4ZZ43JFw}{Wk7-qKrZQQKaoqDC4LglSQ}{192.168.80.3}{192.168.80.3:4300}{http_address=192.168.80.3:4200} elect leader, {cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{Lv_zFA_3QxuAAbxNUNC0RA}{192.168.80.4}{192.168.80.4:4300}{http_address=192.168.80.4:4200} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 18, version: 330, reason: master node changed {previous [], current [{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{Lv_zFA_3QxuAAbxNUNC0RA}{192.168.80.4}{192.168.80.4:4300}{http_address=192.168.80.4:4200}]} | |
| [2022-05-22T23:59:36,086][INFO ][o.e.c.s.ClusterApplierService] [cratedb01] master node changed {previous [], current [{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{Lv_zFA_3QxuAAbxNUNC0RA}{192.168.80.4}{192.168.80.4:4300}{http_address=192.168.80.4:4200}]}, term: 18, version: 330, reason: Publication{term=18, version=330} | |
| [2022-05-22T23:59:36,710][INFO ][o.e.c.s.MasterService ] [cratedb01] node-left[{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{tj6qgjVFS0C0jZfPoaNnBg}{192.168.80.2}{192.168.80.2:4300}{http_address=192.168.80.2:4200} disconnected], term: 18, version: 331, reason: removed {{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{tj6qgjVFS0C0jZfPoaNnBg}{192.168.80.2}{192.168.80.2:4300}{http_address=192.168.80.2:4200},} | |
| [2022-05-22T23:59:36,769][INFO ][o.e.c.s.ClusterApplierService] [cratedb01] removed {{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{tj6qgjVFS0C0jZfPoaNnBg}{192.168.80.2}{192.168.80.2:4300}{http_address=192.168.80.2:4200},}, term: 18, version: 331, reason: Publication{term=18, version=331} | |
| [2022-05-22T23:59:36,815][INFO ][o.e.c.r.DelayedAllocationService] [cratedb01] scheduling reroute for delayed shards in [59.8s] (7 delayed shards) | |
| [2022-05-22T23:59:37,139][INFO ][o.e.i.s.IndexShard ] [cratedb01] [followups][7]primary-replica resync completed with 0 operations | |
| [2022-05-22T23:59:37,265][INFO ][o.e.i.s.IndexShard ] [cratedb01] [followups][4]primary-replica resync completed with 973 operations | |
| [2022-05-22T23:59:37,274][INFO ][o.e.i.s.IndexShard ] [cratedb01] [followups][1]primary-replica resync completed with 1030 operations | |
| [2022-05-22T23:59:37,514][INFO ][o.e.i.s.IndexShard ] [cratedb01] [followups][2]primary-replica resync completed with 1066 operations | |
| [2022-05-22T23:59:37,696][WARN ][o.e.c.r.a.AllocationService] [cratedb01] [followups][9] marking unavailable shards as stale: [sGX2PrCkRfqYkZovvYhoiQ] | |
| [2022-05-22T23:59:38,221][WARN ][o.e.c.r.a.AllocationService] [cratedb01] [followups][3] marking unavailable shards as stale: [bVQWs84TRyKlcfD4s4OZbw] | |
| [2022-05-22T23:59:38,613][WARN ][o.e.c.r.a.AllocationService] [cratedb01] [followups][0] marking unavailable shards as stale: [Uy_fygzXQ6eLknK-17a3Aw] | |
| [2022-05-22T23:59:45,778][INFO ][o.e.c.s.MasterService ] [cratedb01] node-join[{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{o7Sj9n8MQZWf_SgigKb_Gg}{192.168.80.2}{192.168.80.2:4300}{http_address=192.168.80.2:4200} join existing leader], term: 18, version: 335, reason: added {{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{o7Sj9n8MQZWf_SgigKb_Gg}{192.168.80.2}{192.168.80.2:4300}{http_address=192.168.80.2:4200},} | |
| [2022-05-22T23:59:45,869][INFO ][o.e.c.s.ClusterApplierService] [cratedb01] added {{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{o7Sj9n8MQZWf_SgigKb_Gg}{192.168.80.2}{192.168.80.2:4300}{http_address=192.168.80.2:4200},}, term: 18, version: 335, reason: Publication{term=18, version=335} | |
| [2022-05-23T00:00:01,780][WARN ][o.e.c.NodeConnectionsService] [cratedb01] failed to connect to {cratedb03}{4dJJb88aS767BA4ZZ43JFw}{Wk7-qKrZQQKaoqDC4LglSQ}{192.168.80.3}{192.168.80.3:4300}{http_address=192.168.80.3:4200} (tried [1] times) | |
| org.elasticsearch.transport.ConnectTransportException: [cratedb03][192.168.80.3:4300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:961) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$3(ActionListener.java:179) ~[crate-server.jar:?] | |
| at io.crate.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:44) ~[crate-shared.jar:?] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] | |
| at io.crate.concurrent.CompletableContext.addListener(CompletableContext.java:47) ~[crate-shared.jar:?] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.addConnectListener(Netty4TcpChannel.java:103) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.TcpTransport.initiateConnection(TcpTransport.java:322) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.TcpTransport.openConnection(TcpTransport.java:286) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.ClusterConnectionManager.internalOpenConnection(ClusterConnectionManager.java:254) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.ClusterConnectionManager.connectToNode(ClusterConnectionManager.java:139) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:309) ~[crate-server.jar:?] | |
| at org.elasticsearch.cluster.NodeConnectionsService$ConnectionTarget$1.doRun(NodeConnectionsService.java:316) ~[crate-server.jar:?] | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?] | |
| at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] | |
| at org.elasticsearch.cluster.NodeConnectionsService.connectToNodes(NodeConnectionsService.java:139) ~[crate-server.jar:?] | |
| at org.elasticsearch.cluster.service.ClusterApplierService.connectToNodesAndWait(ClusterApplierService.java:482) ~[crate-server.jar:?] | |
| at org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:456) ~[crate-server.jar:?] | |
| at org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:416) ~[crate-server.jar:?] | |
| at org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:163) ~[crate-server.jar:?] | |
| at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:259) ~[crate-server.jar:?] | |
| at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:222) ~[crate-server.jar:?] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?] | |
| at java.lang.Thread.run(Thread.java:833) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 192.168.80.3/192.168.80.3:4300 | |
| Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused | |
| at io.netty.channel.unix.Errors.newConnectException0(Errors.java:155) ~[netty-transport-native-unix-common-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:128) ~[netty-transport-native-unix-common-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.unix.Socket.finishConnect(Socket.java:321) ~[netty-transport-native-unix-common-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:710) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:687) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:477) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:385) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) ~[netty-common-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.76.Final.jar:4.1.76.Final] | |
| ... 1 more | |
| [2022-05-23T00:00:01,796][INFO ][o.e.c.s.MasterService ] [cratedb01] node-left[{cratedb03}{4dJJb88aS767BA4ZZ43JFw}{Wk7-qKrZQQKaoqDC4LglSQ}{192.168.80.3}{192.168.80.3:4300}{http_address=192.168.80.3:4200} disconnected], term: 18, version: 342, reason: removed {{cratedb03}{4dJJb88aS767BA4ZZ43JFw}{Wk7-qKrZQQKaoqDC4LglSQ}{192.168.80.3}{192.168.80.3:4300}{http_address=192.168.80.3:4200},} | |
| [2022-05-23T00:00:01,861][INFO ][o.e.c.s.ClusterApplierService] [cratedb01] removed {{cratedb03}{4dJJb88aS767BA4ZZ43JFw}{Wk7-qKrZQQKaoqDC4LglSQ}{192.168.80.3}{192.168.80.3:4300}{http_address=192.168.80.3:4200},}, term: 18, version: 342, reason: Publication{term=18, version=342} | |
| [2022-05-23T00:00:02,348][INFO ][o.e.i.s.IndexShard ] [cratedb01] [followups][6]primary-replica resync completed with 0 operations | |
| [2022-05-23T00:00:02,397][INFO ][o.e.i.s.IndexShard ] [cratedb01] [followups][8]primary-replica resync completed with 0 operations | |
| [2022-05-23T00:00:02,558][INFO ][o.e.i.s.IndexShard ] [cratedb01] [followups][5]primary-replica resync completed with 0 operations | |
| [2022-05-23T00:00:13,832][INFO ][o.e.c.s.MasterService ] [cratedb01] node-join[{cratedb03}{4dJJb88aS767BA4ZZ43JFw}{bFupXtTPS7uS8ltqH8272w}{192.168.80.3}{192.168.80.3:4300}{http_address=192.168.80.3:4200} join existing leader], term: 18, version: 346, reason: added {{cratedb03}{4dJJb88aS767BA4ZZ43JFw}{bFupXtTPS7uS8ltqH8272w}{192.168.80.3}{192.168.80.3:4300}{http_address=192.168.80.3:4200},} | |
| [2022-05-23T00:00:13,950][INFO ][o.e.c.s.ClusterApplierService] [cratedb01] added {{cratedb03}{4dJJb88aS767BA4ZZ43JFw}{bFupXtTPS7uS8ltqH8272w}{192.168.80.3}{192.168.80.3:4300}{http_address=192.168.80.3:4200},}, term: 18, version: 346, reason: Publication{term=18, version=346} | |
| [2022-05-23T00:00:18,853][INFO ][o.e.c.r.a.AllocationService] [cratedb01] Cluster health status changed from [RED] to [YELLOW] (reason: [shards started [[followups][9]]]). | |
| [2022-05-23T00:00:25,327][INFO ][o.e.c.r.a.AllocationService] [cratedb01] Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[followups][9]]]). |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [2022-05-22T23:59:23,462][WARN ][o.e.t.OutboundHandler ] [cratedb02] send message failed [channel: Netty4TcpChannel{localAddress=/192.168.80.2:4300, remoteAddress=/192.168.80.3:35352}] | |
| java.lang.NullPointerException: Cannot invoke "String.length()" because "str" is null | |
| at org.elasticsearch.common.io.stream.StreamOutput.writeString(StreamOutput.java:359) ~[crate-server.jar:?] | |
| at io.crate.execution.dml.ShardResponse$Failure.writeTo(ShardResponse.java:68) ~[crate-server.jar:?] | |
| at io.crate.execution.dml.ShardResponse.writeTo(ShardResponse.java:210) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.OutboundMessage.writeMessage(OutboundMessage.java:85) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.OutboundMessage.serialize(OutboundMessage.java:62) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.OutboundHandler$MessageSerializer.get(OutboundHandler.java:195) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.OutboundHandler$MessageSerializer.get(OutboundHandler.java:181) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.OutboundHandler$SendContext.get(OutboundHandler.java:231) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.OutboundHandler.internalSend(OutboundHandler.java:162) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.OutboundHandler.sendMessage(OutboundHandler.java:154) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.OutboundHandler.sendResponse(OutboundHandler.java:126) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.TcpTransportChannel.sendResponse(TcpTransportChannel.java:66) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.support.ChannelActionListener.onResponse(ChannelActionListener.java:47) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.support.ChannelActionListener.onResponse(ChannelActionListener.java:30) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishOnSuccess(TransportReplicationAction.java:844) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleResponse(TransportReplicationAction.java:767) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleResponse(TransportReplicationAction.java:753) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.TransportService$TimeoutResponseHandler.handleResponse(TransportService.java:1001) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.TransportService$DirectResponseChannel.processResponse(TransportService.java:1080) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1060) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.support.ChannelActionListener.onResponse(ChannelActionListener.java:47) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.support.ChannelActionListener.onResponse(ChannelActionListener.java:30) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.lambda$runWithPrimaryShardReference$2(TransportReplicationAction.java:415) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:66) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.ActionListener.lambda$map$2(ActionListener.java:163) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:66) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.support.replication.ReplicationOperation.finish(ReplicationOperation.java:351) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.support.replication.ReplicationOperation.decPendingAndFinishIfNeeded(ReplicationOperation.java:332) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.support.replication.ReplicationOperation$2.onResponse(ReplicationOperation.java:216) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.support.replication.ReplicationOperation$2.onResponse(ReplicationOperation.java:209) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.support.RetryableAction$RetryingListener.onResponse(RetryableAction.java:137) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:57) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.TransportService$TimeoutResponseHandler.handleResponse(TransportService.java:1001) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.InboundHandler$1.doRun(InboundHandler.java:207) ~[crate-server.jar:?] | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[crate-server.jar:?] | |
| at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutor.execute(EsExecutors.java:161) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.InboundHandler.handleResponse(InboundHandler.java:199) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.InboundHandler.messageReceived(InboundHandler.java:114) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.InboundHandler.inboundMessage(InboundHandler.java:80) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:687) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:142) [crate-server.jar:?] | |
| at org.elasticsearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:117) [crate-server.jar:?] | |
| at org.elasticsearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:82) [crate-server.jar:?] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:70) [crate-server.jar:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) [netty-handler-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:800) [netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:487) [netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:385) [netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) [netty-common-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.76.Final.jar:4.1.76.Final] | |
| at java.lang.Thread.run(Thread.java:833) [?:?] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [2022-05-22T23:59:35,693][INFO ][o.e.c.c.Coordinator ] [cratedb03] master node [{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{tj6qgjVFS0C0jZfPoaNnBg}{192.168.80.2}{192.168.80.2:4300}{http_address=192.168.80.2:4200}] failed, restarting discovery | |
| org.elasticsearch.transport.NodeDisconnectedException: [cratedb02][192.168.80.2:4300][disconnected] disconnected | |
| [2022-05-22T23:59:35,835][INFO ][o.e.c.s.ClusterApplierService] [cratedb03] master node changed {previous [{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{tj6qgjVFS0C0jZfPoaNnBg}{192.168.80.2}{192.168.80.2:4300}{http_address=192.168.80.2:4200}], current []}, term: 17, version: 329, reason: becoming candidate: onLeaderFailure | |
| [2022-05-22T23:59:35,855][WARN ][o.e.c.NodeConnectionsService] [cratedb03] failed to connect to {cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{tj6qgjVFS0C0jZfPoaNnBg}{192.168.80.2}{192.168.80.2:4300}{http_address=192.168.80.2:4200} (tried [1] times) | |
| org.elasticsearch.transport.ConnectTransportException: [cratedb02][192.168.80.2:4300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:961) ~[crate-server.jar:?] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$3(ActionListener.java:179) ~[crate-server.jar:?] | |
| at io.crate.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:44) ~[crate-shared.jar:?] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] | |
| at io.crate.concurrent.CompletableContext.addListener(CompletableContext.java:47) ~[crate-shared.jar:?] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.addConnectListener(Netty4TcpChannel.java:103) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.TcpTransport.initiateConnection(TcpTransport.java:322) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.TcpTransport.openConnection(TcpTransport.java:286) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.ClusterConnectionManager.internalOpenConnection(ClusterConnectionManager.java:254) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.ClusterConnectionManager.connectToNode(ClusterConnectionManager.java:139) ~[crate-server.jar:?] | |
| at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:309) ~[crate-server.jar:?] | |
| at org.elasticsearch.cluster.NodeConnectionsService$ConnectionTarget$1.doRun(NodeConnectionsService.java:316) ~[crate-server.jar:?] | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?] | |
| at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] | |
| at org.elasticsearch.cluster.NodeConnectionsService.connectToNodes(NodeConnectionsService.java:139) ~[crate-server.jar:?] | |
| at org.elasticsearch.cluster.service.ClusterApplierService.connectToNodesAndWait(ClusterApplierService.java:482) ~[crate-server.jar:?] | |
| at org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:456) ~[crate-server.jar:?] | |
| at org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:416) ~[crate-server.jar:?] | |
| at org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:163) ~[crate-server.jar:?] | |
| at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:259) ~[crate-server.jar:?] | |
| at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:222) ~[crate-server.jar:?] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?] | |
| at java.lang.Thread.run(Thread.java:833) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 192.168.80.2/192.168.80.2:4300 | |
| Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused | |
| at io.netty.channel.unix.Errors.newConnectException0(Errors.java:155) ~[netty-transport-native-unix-common-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:128) ~[netty-transport-native-unix-common-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.unix.Socket.finishConnect(Socket.java:321) ~[netty-transport-native-unix-common-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:710) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:687) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:477) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:385) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) ~[netty-common-4.1.76.Final.jar:4.1.76.Final] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.76.Final.jar:4.1.76.Final] | |
| ... 1 more | |
| [2022-05-22T23:59:36,066][INFO ][o.e.c.s.ClusterApplierService] [cratedb03] master node changed {previous [], current [{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{Lv_zFA_3QxuAAbxNUNC0RA}{192.168.80.4}{192.168.80.4:4300}{http_address=192.168.80.4:4200}]}, term: 18, version: 330, reason: ApplyCommitRequest{term=18, version=330, sourceNode={cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{Lv_zFA_3QxuAAbxNUNC0RA}{192.168.80.4}{192.168.80.4:4300}{http_address=192.168.80.4:4200}} | |
| [2022-05-22T23:59:36,745][INFO ][o.e.c.s.ClusterApplierService] [cratedb03] removed {{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{tj6qgjVFS0C0jZfPoaNnBg}{192.168.80.2}{192.168.80.2:4300}{http_address=192.168.80.2:4200},}, term: 18, version: 331, reason: ApplyCommitRequest{term=18, version=331, sourceNode={cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{Lv_zFA_3QxuAAbxNUNC0RA}{192.168.80.4}{192.168.80.4:4300}{http_address=192.168.80.4:4200}} | |
| [2022-05-22T23:59:37,078][INFO ][o.e.i.s.IndexShard ] [cratedb03] [followups][9]primary-replica resync completed with 975 operations | |
| [2022-05-22T23:59:38,872][INFO ][o.e.i.s.IndexShard ] [cratedb03] [followups][3]primary-replica resync completed with 2010 operations | |
| [2022-05-22T23:59:38,873][INFO ][o.e.i.s.IndexShard ] [cratedb03] [followups][0]primary-replica resync completed with 1970 operations | |
| [2022-05-22T23:59:45,796][INFO ][o.e.c.s.ClusterApplierService] [cratedb03] added {{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{o7Sj9n8MQZWf_SgigKb_Gg}{192.168.80.2}{192.168.80.2:4300}{http_address=192.168.80.2:4200},}, term: 18, version: 335, reason: ApplyCommitRequest{term=18, version=335, sourceNode={cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{Lv_zFA_3QxuAAbxNUNC0RA}{192.168.80.4}{192.168.80.4:4300}{http_address=192.168.80.4:4200}} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment