No logs, same as before.
-
-
Save rafbgarcia/a31e561b5c12ba87cce466a2c3919039 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| version: '3.8' | |
| services: | |
| cratedb01: | |
| image: crate/crate:nightly | |
| ports: | |
| - "4201:4200" | |
| volumes: | |
| - /tmp/crate/01:/data | |
| command: ["crate", | |
| "-Ccluster.name=crate-docker-cluster", | |
| "-Cnode.name=cratedb01", | |
| "-Cnode.data=true", | |
| "-Cnetwork.host=_site_", | |
| "-Cdiscovery.seed_hosts=cratedb02,cratedb03", | |
| "-Ccluster.initial_master_nodes=cratedb01,cratedb02,cratedb03", | |
| "-Cgateway.expected_nodes=3", | |
| "-Cgateway.recover_after_nodes=2"] | |
| deploy: | |
| replicas: 1 | |
| restart_policy: | |
| condition: on-failure | |
| environment: | |
| - CRATE_HEAP_SIZE=2g | |
| cratedb02: | |
| image: crate/crate:nightly | |
| ports: | |
| - "4202:4200" | |
| volumes: | |
| - /tmp/crate/02:/data | |
| command: ["crate", | |
| "-Ccluster.name=crate-docker-cluster", | |
| "-Cnode.name=cratedb02", | |
| "-Cnode.data=true", | |
| "-Cnetwork.host=_site_", | |
| "-Cdiscovery.seed_hosts=cratedb01,cratedb03", | |
| "-Ccluster.initial_master_nodes=cratedb01,cratedb02,cratedb03", | |
| "-Cgateway.expected_nodes=3", | |
| "-Cgateway.recover_after_nodes=2"] | |
| deploy: | |
| replicas: 1 | |
| restart_policy: | |
| condition: on-failure | |
| environment: | |
| - CRATE_HEAP_SIZE=2g | |
| cratedb03: | |
| image: crate/crate:nightly | |
| ports: | |
| - "4203:4200" | |
| volumes: | |
| - /tmp/crate/03:/data | |
| command: ["crate", | |
| "-Ccluster.name=crate-docker-cluster", | |
| "-Cnode.name=cratedb03", | |
| "-Cnode.data=true", | |
| "-Cnetwork.host=_site_", | |
| "-Cdiscovery.seed_hosts=cratedb01,cratedb02", | |
| "-Ccluster.initial_master_nodes=cratedb01,cratedb02,cratedb03", | |
| "-Cgateway.expected_nodes=3", | |
| "-Cgateway.recover_after_nodes=2"] | |
| deploy: | |
| replicas: 1 | |
| restart_policy: | |
| condition: on-failure | |
| environment: | |
| - CRATE_HEAP_SIZE=2g |
[2022-05-23T14:45:15,233][INFO ][o.e.c.c.Coordinator ] [cratedb02] master node [{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{PCxqmcR3SCGdHGFt5WR8Ww}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200}] failed, restarting discovery
org.elasticsearch.transport.NodeDisconnectedException: [cratedb01][172.21.0.4:4300][disconnected] disconnected
[2022-05-23T14:45:15,321][INFO ][o.e.c.s.ClusterApplierService] [cratedb02] master node changed {previous [{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{PCxqmcR3SCGdHGFt5WR8Ww}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200}], current []}, term: 72, version: 1292, reason: becoming candidate: onLeaderFailure
[2022-05-23T14:45:15,329][WARN ][o.e.c.NodeConnectionsService] [cratedb02] failed to connect to {cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{PCxqmcR3SCGdHGFt5WR8Ww}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200} (tried [1] times)
org.elasticsearch.transport.ConnectTransportException: [cratedb01][172.21.0.4:4300] connect_exception
at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:961) ~[crate-server.jar:?]
at org.elasticsearch.action.ActionListener.accept(ActionListener.java:56) ~[crate-server.jar:?]
at org.elasticsearch.action.ActionListener.accept(ActionListener.java:39) ~[crate-server.jar:?]
at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?]
at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?]
at org.elasticsearch.transport.netty4.Netty4TcpChannel.addConnectListener(Netty4TcpChannel.java:99) ~[crate-server.jar:?]
at org.elasticsearch.transport.TcpTransport.initiateConnection(TcpTransport.java:322) ~[crate-server.jar:?]
at org.elasticsearch.transport.TcpTransport.openConnection(TcpTransport.java:286) ~[crate-server.jar:?]
at org.elasticsearch.transport.ClusterConnectionManager.internalOpenConnection(ClusterConnectionManager.java:254) ~[crate-server.jar:?]
at org.elasticsearch.transport.ClusterConnectionManager.connectToNode(ClusterConnectionManager.java:139) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:309) ~[crate-server.jar:?]
at org.elasticsearch.cluster.NodeConnectionsService$ConnectionTarget$1.doRun(NodeConnectionsService.java:316) ~[crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?]
at org.elasticsearch.cluster.NodeConnectionsService.connectToNodes(NodeConnectionsService.java:139) ~[crate-server.jar:?]
at org.elasticsearch.cluster.service.ClusterApplierService.connectToNodesAndWait(ClusterApplierService.java:482) ~[crate-server.jar:?]
at org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:456) ~[crate-server.jar:?]
at org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:416) ~[crate-server.jar:?]
at org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:163) ~[crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:259) ~[crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:222) ~[crate-server.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 172.21.0.4/172.21.0.4:4300
Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused
at io.netty.channel.unix.Errors.newConnectException0(Errors.java:155) ~[netty-transport-native-unix-common-4.1.76.Final.jar:4.1.76.Final]
at io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:128) ~[netty-transport-native-unix-common-4.1.76.Final.jar:4.1.76.Final]
at io.netty.channel.unix.Socket.finishConnect(Socket.java:321) ~[netty-transport-native-unix-common-4.1.76.Final.jar:4.1.76.Final]
at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:710) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final]
at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:687) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final]
at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final]
at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:477) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final]
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:385) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) ~[netty-common-4.1.76.Final.jar:4.1.76.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.76.Final.jar:4.1.76.Final]
... 1 more
[2022-05-23T14:45:16,289][INFO ][o.e.c.s.MasterService ] [cratedb02] elected-as-master ([2] nodes joined)[{cratedb03}{4dJJb88aS767BA4ZZ43JFw}{FC8DqdwjRwO-wpfjy8yIAA}{172.21.0.2}{172.21.0.2:4300}{http_address=172.21.0.2:4200} elect leader, {cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{fpFiKtiLRhSjWvoYfYOGNg}{172.21.0.3}{172.21.0.3:4300}{http_address=172.21.0.3:4200} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 75, version: 1293, reason: master node changed {previous [], current [{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{fpFiKtiLRhSjWvoYfYOGNg}{172.21.0.3}{172.21.0.3:4300}{http_address=172.21.0.3:4200}]}
[2022-05-23T14:45:16,430][INFO ][o.e.c.s.ClusterApplierService] [cratedb02] master node changed {previous [], current [{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{fpFiKtiLRhSjWvoYfYOGNg}{172.21.0.3}{172.21.0.3:4300}{http_address=172.21.0.3:4200}]}, term: 75, version: 1293, reason: Publication{term=75, version=1293}
[2022-05-23T14:45:16,592][INFO ][o.e.c.s.MasterService ] [cratedb02] node-left[{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{PCxqmcR3SCGdHGFt5WR8Ww}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200} disconnected], term: 75, version: 1294, reason: removed {{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{PCxqmcR3SCGdHGFt5WR8Ww}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200},}
[2022-05-23T14:45:16,617][INFO ][o.e.c.s.ClusterApplierService] [cratedb02] removed {{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{PCxqmcR3SCGdHGFt5WR8Ww}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200},}, term: 75, version: 1294, reason: Publication{term=75, version=1294}
[2022-05-23T14:45:16,621][INFO ][o.e.c.r.DelayedAllocationService] [cratedb02] scheduling reroute for delayed shards in [59.8s] (4 delayed shards)
[2022-05-23T14:45:16,635][WARN ][i.c.e.d.u.TransportShardUpsertAction] [cratedb02] [[followups][1]] failed to perform internal:crate:sql/data/write on replica [followups][1], node[onjmxXu0Q5mGvEZqBYve8A], [R], s[STARTED], a[id=cwVzX2bRRiWx2CYuUf5lww]
org.elasticsearch.client.transport.NoNodeAvailableException: unknown node [onjmxXu0Q5mGvEZqBYve8A]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1023) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
[2022-05-23T14:45:16,676][WARN ][o.e.c.r.a.AllocationService] [cratedb02] [followups][5] marking unavailable shards as stale: [jgLxoq6oRdWGBlej30flMw]
[2022-05-23T14:45:16,678][WARN ][o.e.c.r.a.AllocationService] [cratedb02] [followups][1] marking unavailable shards as stale: [cwVzX2bRRiWx2CYuUf5lww]
[2022-05-23T14:45:17,291][WARN ][o.e.c.r.a.AllocationService] [cratedb02] [followups][3] marking unavailable shards as stale: [dJlfuwF-Qn2o4I7XG00fUA]
[2022-05-23T14:45:18,411][WARN ][i.c.e.d.u.TransportShardUpsertAction] [cratedb02] [[followups][2]] failed to perform internal:crate:sql/data/write on replica [followups][2], node[onjmxXu0Q5mGvEZqBYve8A], [R], s[STARTED], a[id=TI27JboWSZuv5S8OErYbzQ]
org.elasticsearch.client.transport.NoNodeAvailableException: unknown node [onjmxXu0Q5mGvEZqBYve8A]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1023) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
[2022-05-23T14:45:18,415][WARN ][o.e.c.r.a.AllocationService] [cratedb02] [followups][2] marking unavailable shards as stale: [TI27JboWSZuv5S8OErYbzQ]
[2022-05-23T14:45:26,377][INFO ][o.e.c.s.MasterService ] [cratedb02] node-join[{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{5EohfyDESoGHVFL44x5akw}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200} join existing leader], term: 75, version: 1298, reason: added {{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{5EohfyDESoGHVFL44x5akw}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200},}
[2022-05-23T14:45:26,443][INFO ][o.e.c.s.ClusterApplierService] [cratedb02] added {{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{5EohfyDESoGHVFL44x5akw}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200},}, term: 75, version: 1298, reason: Publication{term=75, version=1298}
[2022-05-23T14:46:13,423][INFO ][o.e.c.s.MasterService ] [cratedb02] node-left[{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{5EohfyDESoGHVFL44x5akw}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200} disconnected], term: 75, version: 1304, reason: removed {{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{5EohfyDESoGHVFL44x5akw}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200},}
[2022-05-23T14:46:13,443][INFO ][o.e.c.s.ClusterApplierService] [cratedb02] removed {{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{5EohfyDESoGHVFL44x5akw}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200},}, term: 75, version: 1304, reason: Publication{term=75, version=1304}
[2022-05-23T14:46:13,449][INFO ][o.e.c.r.DelayedAllocationService] [cratedb02] scheduling reroute for delayed shards in [59.9s] (4 delayed shards)
[2022-05-23T14:46:21,718][INFO ][o.e.c.s.MasterService ] [cratedb02] node-join[{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{_2ejReLvRhmURiZAMaBavQ}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200} join existing leader], term: 75, version: 1305, reason: added {{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{_2ejReLvRhmURiZAMaBavQ}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200},}
[2022-05-23T14:46:21,766][INFO ][o.e.c.s.ClusterApplierService] [cratedb02] added {{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{_2ejReLvRhmURiZAMaBavQ}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200},}, term: 75, version: 1305, reason: Publication{term=75, version=1305}
[2022-05-23T14:46:45,103][INFO ][o.e.c.r.a.AllocationService] [cratedb02] Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[followups][5]]]).
[2022-05-23T14:45:15,223][INFO ][o.e.c.c.Coordinator ] [cratedb03] master node [{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{PCxqmcR3SCGdHGFt5WR8Ww}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200}] failed, restarting discovery
org.elasticsearch.transport.NodeDisconnectedException: [cratedb01][172.21.0.4:4300][disconnected] disconnected
[2022-05-23T14:45:15,234][INFO ][o.e.c.s.ClusterApplierService] [cratedb03] master node changed {previous [{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{PCxqmcR3SCGdHGFt5WR8Ww}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200}], current []}, term: 72, version: 1292, reason: becoming candidate: onLeaderFailure
[2022-05-23T14:45:15,289][WARN ][o.e.c.NodeConnectionsService] [cratedb03] failed to connect to {cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{PCxqmcR3SCGdHGFt5WR8Ww}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200} (tried [1] times)
org.elasticsearch.transport.ConnectTransportException: [cratedb01][172.21.0.4:4300] connect_exception
at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:961) ~[crate-server.jar:?]
at org.elasticsearch.action.ActionListener.accept(ActionListener.java:56) ~[crate-server.jar:?]
at org.elasticsearch.action.ActionListener.accept(ActionListener.java:39) ~[crate-server.jar:?]
at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?]
at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?]
at org.elasticsearch.transport.netty4.Netty4TcpChannel.addConnectListener(Netty4TcpChannel.java:99) ~[crate-server.jar:?]
at org.elasticsearch.transport.TcpTransport.initiateConnection(TcpTransport.java:322) ~[crate-server.jar:?]
at org.elasticsearch.transport.TcpTransport.openConnection(TcpTransport.java:286) ~[crate-server.jar:?]
at org.elasticsearch.transport.ClusterConnectionManager.internalOpenConnection(ClusterConnectionManager.java:254) ~[crate-server.jar:?]
at org.elasticsearch.transport.ClusterConnectionManager.connectToNode(ClusterConnectionManager.java:139) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:309) ~[crate-server.jar:?]
at org.elasticsearch.cluster.NodeConnectionsService$ConnectionTarget$1.doRun(NodeConnectionsService.java:316) ~[crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?]
at org.elasticsearch.cluster.NodeConnectionsService.connectToNodes(NodeConnectionsService.java:139) ~[crate-server.jar:?]
at org.elasticsearch.cluster.service.ClusterApplierService.connectToNodesAndWait(ClusterApplierService.java:482) ~[crate-server.jar:?]
at org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:456) ~[crate-server.jar:?]
at org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:416) ~[crate-server.jar:?]
at org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:163) ~[crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:259) ~[crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:222) ~[crate-server.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 172.21.0.4/172.21.0.4:4300
Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused
at io.netty.channel.unix.Errors.newConnectException0(Errors.java:155) ~[netty-transport-native-unix-common-4.1.76.Final.jar:4.1.76.Final]
at io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:128) ~[netty-transport-native-unix-common-4.1.76.Final.jar:4.1.76.Final]
at io.netty.channel.unix.Socket.finishConnect(Socket.java:321) ~[netty-transport-native-unix-common-4.1.76.Final.jar:4.1.76.Final]
at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:710) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final]
at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:687) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final]
at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final]
at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:477) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final]
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:385) ~[netty-transport-classes-epoll-4.1.76.Final.jar:4.1.76.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) ~[netty-common-4.1.76.Final.jar:4.1.76.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.76.Final.jar:4.1.76.Final]
... 1 more
[2022-05-23T14:45:16,408][INFO ][o.e.c.s.ClusterApplierService] [cratedb03] master node changed {previous [], current [{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{fpFiKtiLRhSjWvoYfYOGNg}{172.21.0.3}{172.21.0.3:4300}{http_address=172.21.0.3:4200}]}, term: 75, version: 1293, reason: ApplyCommitRequest{term=75, version=1293, sourceNode={cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{fpFiKtiLRhSjWvoYfYOGNg}{172.21.0.3}{172.21.0.3:4300}{http_address=172.21.0.3:4200}}
[2022-05-23T14:45:16,607][INFO ][o.e.c.s.ClusterApplierService] [cratedb03] removed {{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{PCxqmcR3SCGdHGFt5WR8Ww}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200},}, term: 75, version: 1294, reason: ApplyCommitRequest{term=75, version=1294, sourceNode={cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{fpFiKtiLRhSjWvoYfYOGNg}{172.21.0.3}{172.21.0.3:4300}{http_address=172.21.0.3:4200}}
[2022-05-23T14:45:16,652][WARN ][i.c.e.d.u.TransportShardUpsertAction] [cratedb03] [[followups][5]] failed to perform internal:crate:sql/data/write on replica [followups][5], node[onjmxXu0Q5mGvEZqBYve8A], [R], s[STARTED], a[id=jgLxoq6oRdWGBlej30flMw]
org.elasticsearch.client.transport.NoNodeAvailableException: unknown node [onjmxXu0Q5mGvEZqBYve8A]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1023) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
[2022-05-23T14:45:17,282][WARN ][i.c.e.d.u.TransportShardUpsertAction] [cratedb03] [[followups][3]] failed to perform internal:crate:sql/data/write on replica [followups][3], node[onjmxXu0Q5mGvEZqBYve8A], [R], s[STARTED], a[id=dJlfuwF-Qn2o4I7XG00fUA]
org.elasticsearch.client.transport.NoNodeAvailableException: unknown node [onjmxXu0Q5mGvEZqBYve8A]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1023) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
[2022-05-23T14:45:26,387][INFO ][o.e.c.s.ClusterApplierService] [cratedb03] added {{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{5EohfyDESoGHVFL44x5akw}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200},}, term: 75, version: 1298, reason: ApplyCommitRequest{term=75, version=1298, sourceNode={cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{fpFiKtiLRhSjWvoYfYOGNg}{172.21.0.3}{172.21.0.3:4300}{http_address=172.21.0.3:4200}}
[2022-05-23T14:46:13,439][INFO ][o.e.c.s.ClusterApplierService] [cratedb03] removed {{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{5EohfyDESoGHVFL44x5akw}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200},}, term: 75, version: 1304, reason: ApplyCommitRequest{term=75, version=1304, sourceNode={cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{fpFiKtiLRhSjWvoYfYOGNg}{172.21.0.3}{172.21.0.3:4300}{http_address=172.21.0.3:4200}}
[2022-05-23T14:46:21,723][INFO ][o.e.c.s.ClusterApplierService] [cratedb03] added {{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{_2ejReLvRhmURiZAMaBavQ}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200},}, term: 75, version: 1305, reason: ApplyCommitRequest{term=75, version=1305, sourceNode={cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{fpFiKtiLRhSjWvoYfYOGNg}{172.21.0.3}{172.21.0.3:4300}{http_address=172.21.0.3:4200}}
[2022-05-23T14:56:35,774][INFO ][o.e.c.s.ClusterApplierService] [cratedb01] removed {{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{dRk5MZ8VRjaqTYoB38JfAA}{172.21.0.3}{172.21.0.3:4300}{http_address=172.21.0.3:4200},}, term: 77, version: 1349, reason: ApplyCommitRequest{term=77, version=1349, sourceNode={cratedb03}{4dJJb88aS767BA4ZZ43JFw}{8p3N6AW5T6mfjRwb0-T72Q}{172.21.0.2}{172.21.0.2:4300}{http_address=172.21.0.2:4200}}
[2022-05-23T14:56:35,832][WARN ][i.c.e.d.u.TransportShardUpsertAction] [cratedb01] [[followups][1]] failed to perform internal:crate:sql/data/write on replica [followups][1], node[BvpxgumpQpyUDDdaVS_7Tw], [R], s[STARTED], a[id=n4Fi2xYsTTyO2ai5v7gN_Q]
org.elasticsearch.client.transport.NoNodeAvailableException: unknown node [BvpxgumpQpyUDDdaVS_7Tw]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1023) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeDisconnectedException: [cratedb02][172.21.0.3:4300][internal:crate:sql/data/write[r]] disconnected
[2022-05-23T14:56:35,861][WARN ][i.c.e.d.u.TransportShardUpsertAction] [cratedb01] [[followups][2]] failed to perform internal:crate:sql/data/write on replica [followups][2], node[BvpxgumpQpyUDDdaVS_7Tw], [R], s[STARTED], a[id=CL9QKeHpQ5OTMhwg03sgJw]
org.elasticsearch.client.transport.NoNodeAvailableException: unknown node [BvpxgumpQpyUDDdaVS_7Tw]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1023) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeDisconnectedException: [cratedb02][172.21.0.3:4300][internal:crate:sql/data/write[r]] disconnected
[2022-05-23T14:56:48,082][INFO ][o.e.c.s.ClusterApplierService] [cratedb01] added {{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{Pr7U5ecHTTaRGCmSuizpKw}{172.21.0.3}{172.21.0.3:4300}{http_address=172.21.0.3:4200},}, term: 77, version: 1354, reason: ApplyCommitRequest{term=77, version=1354, sourceNode={cratedb03}{4dJJb88aS767BA4ZZ43JFw}{8p3N6AW5T6mfjRwb0-T72Q}{172.21.0.2}{172.21.0.2:4300}{http_address=172.21.0.2:4200}}
Disconnected. No logs.
[2022-05-23T14:56:35,748][INFO ][o.e.c.s.MasterService ] [cratedb03] node-left[{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{dRk5MZ8VRjaqTYoB38JfAA}{172.21.0.3}{172.21.0.3:4300}{http_address=172.21.0.3:4200} disconnected], term: 77, version: 1349, reason: removed {{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{dRk5MZ8VRjaqTYoB38JfAA}{172.21.0.3}{172.21.0.3:4300}{http_address=172.21.0.3:4200},}
[2022-05-23T14:56:35,779][INFO ][o.e.c.s.ClusterApplierService] [cratedb03] removed {{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{dRk5MZ8VRjaqTYoB38JfAA}{172.21.0.3}{172.21.0.3:4300}{http_address=172.21.0.3:4200},}, term: 77, version: 1349, reason: Publication{term=77, version=1349}
[2022-05-23T14:56:35,785][INFO ][o.e.c.r.DelayedAllocationService] [cratedb03] scheduling reroute for delayed shards in [59.9s] (4 delayed shards)
[2022-05-23T14:56:35,864][WARN ][o.e.c.r.a.AllocationService] [cratedb03] [followups][1] marking unavailable shards as stale: [n4Fi2xYsTTyO2ai5v7gN_Q]
[2022-05-23T14:56:35,915][WARN ][o.e.c.r.a.AllocationService] [cratedb03] [followups][2] marking unavailable shards as stale: [CL9QKeHpQ5OTMhwg03sgJw]
[2022-05-23T14:56:36,433][INFO ][o.e.i.s.IndexShard ] [cratedb03] [followups][0]primary-replica resync completed with 0 operations
[2022-05-23T14:56:36,447][INFO ][o.e.i.s.IndexShard ] [cratedb03] [followups][4]primary-replica resync completed with 1714 operations
[2022-05-23T14:56:36,630][WARN ][o.e.c.r.a.AllocationService] [cratedb03] [followups][4] marking unavailable shards as stale: [hvj8gkpzTHWg_WzPnISuIA]
[2022-05-23T14:56:37,141][WARN ][o.e.c.r.a.AllocationService] [cratedb03] [followups][0] marking unavailable shards as stale: [HF7QlZLcSd6YhGRLcIN6cQ]
[2022-05-23T14:56:48,064][INFO ][o.e.c.s.MasterService ] [cratedb03] node-join[{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{Pr7U5ecHTTaRGCmSuizpKw}{172.21.0.3}{172.21.0.3:4300}{http_address=172.21.0.3:4200} join existing leader], term: 77, version: 1354, reason: added {{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{Pr7U5ecHTTaRGCmSuizpKw}{172.21.0.3}{172.21.0.3:4300}{http_address=172.21.0.3:4200},}
[2022-05-23T14:56:48,239][INFO ][o.e.c.s.ClusterApplierService] [cratedb03] added {{cratedb02}{BvpxgumpQpyUDDdaVS_7Tw}{Pr7U5ecHTTaRGCmSuizpKw}{172.21.0.3}{172.21.0.3:4300}{http_address=172.21.0.3:4200},}, term: 77, version: 1354, reason: Publication{term=77, version=1354}
[2022-05-23T14:57:14,900][WARN ][o.e.m.j.JvmGcMonitorService] [cratedb03] [gc][203] overhead, spent [819ms] collecting in the last [1.3s]
[2022-05-23T14:57:15,351][INFO ][o.e.c.s.MasterService ] [cratedb03] node-left[{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{qjpLLK9-TP-XtnZEc0ZAQg}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200} disconnected], term: 77, version: 1361, reason: removed {{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{qjpLLK9-TP-XtnZEc0ZAQg}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200},}
[2022-05-23T14:57:15,586][INFO ][o.e.c.s.ClusterApplierService] [cratedb03] removed {{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{qjpLLK9-TP-XtnZEc0ZAQg}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200},}, term: 77, version: 1361, reason: Publication{term=77, version=1361}
[2022-05-23T14:57:15,589][INFO ][o.e.c.r.DelayedAllocationService] [cratedb03] scheduling reroute for delayed shards in [59.7s] (4 delayed shards)
[2022-05-23T14:57:15,765][WARN ][i.c.e.d.u.TransportShardUpsertAction] [cratedb03] [[followups][5]] failed to perform internal:crate:sql/data/write on replica [followups][5], node[onjmxXu0Q5mGvEZqBYve8A], [R], s[STARTED], a[id=aeoMo49_QnKtRVu2q4-O2g]
org.elasticsearch.client.transport.NoNodeAvailableException: unknown node [onjmxXu0Q5mGvEZqBYve8A]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1023) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
[2022-05-23T14:57:15,775][WARN ][o.e.c.r.a.AllocationService] [cratedb03] [followups][5] marking unavailable shards as stale: [aeoMo49_QnKtRVu2q4-O2g]
[2022-05-23T14:57:15,875][WARN ][o.e.t.OutboundHandler ] [cratedb03] send message failed [channel: Netty4TcpChannel{localAddress=/172.21.0.2:4300, remoteAddress=/172.21.0.4:54908}]
io.netty.channel.StacklessClosedChannelException: null
at io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[netty-transport-4.1.76.Final.jar:4.1.76.Final]
[2022-05-23T14:57:15,895][WARN ][i.c.e.d.u.TransportShardUpsertAction] [cratedb03] [[followups][3]] failed to perform internal:crate:sql/data/write on replica [followups][3], node[onjmxXu0Q5mGvEZqBYve8A], [R], s[STARTED], a[id=9fv5zcibSISyKWhk-LmjOA]
org.elasticsearch.client.transport.NoNodeAvailableException: unknown node [onjmxXu0Q5mGvEZqBYve8A]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1023) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Suppressed: org.elasticsearch.transport.NodeNotConnectedException: [cratedb01][172.21.0.4:4300] Node not connected
at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:189) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:525) ~[crate-server.jar:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:496) ~[crate-server.jar:?]
at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1029) [crate-server.jar:?]
at org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:248) [crate-server.jar:?]
at org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:100) [crate-server.jar:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [crate-server.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
[2022-05-23T14:57:15,919][WARN ][o.e.c.r.a.AllocationService] [cratedb03] [followups][3] marking unavailable shards as stale: [9fv5zcibSISyKWhk-LmjOA]
[2022-05-23T14:57:16,116][WARN ][o.e.t.OutboundHandler ] [cratedb03] send message failed [channel: Netty4TcpChannel{localAddress=/172.21.0.2:4300, remoteAddress=/172.21.0.4:54898}]
io.netty.channel.StacklessClosedChannelException: null
at io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[netty-transport-4.1.76.Final.jar:4.1.76.Final]
[2022-05-23T14:57:32,959][INFO ][o.e.c.s.MasterService ] [cratedb03] node-join[{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{bQl65OfHQ5CJlzwWhuAwXw}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200} join existing leader], term: 77, version: 1366, reason: added {{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{bQl65OfHQ5CJlzwWhuAwXw}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200},}
[2022-05-23T14:57:33,277][INFO ][o.e.c.s.ClusterApplierService] [cratedb03] added {{cratedb01}{onjmxXu0Q5mGvEZqBYve8A}{bQl65OfHQ5CJlzwWhuAwXw}{172.21.0.4}{172.21.0.4:4300}{http_address=172.21.0.4:4200},}, term: 77, version: 1366, reason: Publication{term=77, version=1366}
[2022-05-23T14:57:46,859][INFO ][o.e.c.r.a.AllocationService] [cratedb03] Cluster health status changed from [RED] to [YELLOW] (reason: [shards started [[followups][1]]]).
[2022-05-23T14:57:57,037][INFO ][o.e.c.r.a.AllocationService] [cratedb03] Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[followups][3]]]).
Removed server http://localhost:4201 from active pool
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "/usr/local/Cellar/python@3.9/3.9.9/Frameworks/Python.framework/Versions/3.9/lib/python3.9/http/client.py", line 1377, in getresponse
response.begin()
File "/usr/local/Cellar/python@3.9/3.9.9/Frameworks/Python.framework/Versions/3.9/lib/python3.9/http/client.py", line 320, in begin
version, status, reason = self._read_status()
File "/usr/local/Cellar/python@3.9/3.9.9/Frameworks/Python.framework/Versions/3.9/lib/python3.9/http/client.py", line 289, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/crate/client/http.py", line 473, in _request
response = self.server_pool[next_server].request(
File "/usr/local/lib/python3.9/site-packages/crate/client/http.py", line 154, in request
return self.pool.urlopen(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/retry.py", line 532, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/local/lib/python3.9/site-packages/urllib3/packages/six.py", line 769, in reraise
raise value.with_traceback(tb)
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "/usr/local/Cellar/python@3.9/3.9.9/Frameworks/Python.framework/Versions/3.9/lib/python3.9/http/client.py", line 1377, in getresponse
response.begin()
File "/usr/local/Cellar/python@3.9/3.9.9/Frameworks/Python.framework/Versions/3.9/lib/python3.9/http/client.py", line 320, in begin
version, status, reason = self._read_status()
File "/usr/local/Cellar/python@3.9/3.9.9/Frameworks/Python.framework/Versions/3.9/lib/python3.9/http/client.py", line 289, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/rafa/repos/crate/test.py", line 29, in <module>
cratecur.execute("""
File "/usr/local/lib/python3.9/site-packages/crate/client/cursor.py", line 53, in execute
self._result = self.connection.client.sql(sql, parameters,
File "/usr/local/lib/python3.9/site-packages/crate/client/http.py", line 396, in sql
content = self._json_request('POST', self.path, data=data)
File "/usr/local/lib/python3.9/site-packages/crate/client/http.py", line 522, in _json_request
response = self._request(method, path, data=data)
File "/usr/local/lib/python3.9/site-packages/crate/client/http.py", line 513, in _request
self._drop_server(next_server, ex_message)
File "/usr/local/lib/python3.9/site-packages/crate/client/http.py", line 581, in _drop_server
raise ConnectionError(
crate.client.exceptions.ConnectionError: No more Servers available, exception from last server: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
78.85 real 6.80 user 0.75 sys
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment